View Javadoc

1   /*
2    * Copyright 2015 Data Archiving and Networked Services (an institute of
3    * Koninklijke Nederlandse Akademie van Wetenschappen), King's College London,
4    * Georg-August-Universitaet Goettingen Stiftung Oeffentlichen Rechts
5    *
6    * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by
7    * the European Commission - subsequent versions of the EUPL (the "Licence");
8    * You may not use this work except in compliance with the Licence.
9    * You may obtain a copy of the Licence at:
10   *
11   * https://joinup.ec.europa.eu/software/page/eupl
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the Licence is distributed on an "AS IS" basis,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the Licence for the specific language governing
17   * permissions and limitations under the Licence.
18   */
19  
20  package eu.ehri.project.commands;
21  
22  import com.google.common.base.Charsets;
23  import com.google.common.collect.Lists;
24  import com.tinkerpop.frames.FramedGraph;
25  import eu.ehri.project.acl.SystemScope;
26  import eu.ehri.project.core.GraphManager;
27  import eu.ehri.project.core.GraphManagerFactory;
28  import eu.ehri.project.importers.ImportCallback;
29  import eu.ehri.project.importers.ImportLog;
30  import eu.ehri.project.importers.base.ItemImporter;
31  import eu.ehri.project.importers.base.SaxXmlHandler;
32  import eu.ehri.project.importers.managers.SaxImportManager;
33  import eu.ehri.project.importers.properties.XmlImportProperties;
34  import eu.ehri.project.models.UserProfile;
35  import eu.ehri.project.models.base.PermissionScope;
36  import org.apache.commons.cli.CommandLine;
37  import org.apache.commons.cli.Option;
38  import org.apache.commons.cli.Options;
39  
40  import java.io.BufferedReader;
41  import java.io.InputStream;
42  import java.io.InputStreamReader;
43  import java.nio.file.Files;
44  import java.nio.file.Paths;
45  import java.util.List;
46  import java.util.Map;
47  
48  /**
49   * Superclass of all import command-line tools.
50   */
51  public abstract class ImportCommand extends BaseCommand {
52      private final Class<? extends SaxXmlHandler> handler;
53      private final Class<? extends ItemImporter> importer;
54  
55      public ImportCommand(Class<? extends SaxXmlHandler> handler, Class<? extends ItemImporter> importer) {
56          this.handler = handler;
57          this.importer = importer;
58      }
59  
60      @Override
61      protected void setCustomOptions(Options options) {
62          options.addOption(Option.builder()
63                  .longOpt("scope")
64                  .hasArg()
65                  .required()
66                  .type(String.class)
67                  .desc("Identifier of scope to import into, i.e. repository")
68                  .build());
69          options.addOption(Option.builder("F")
70                  .longOpt("files-from")
71                  .hasArg()
72                  .type(String.class)
73                  .desc("Read list of input files from another file (or standard input, if given '-')")
74                  .build());
75          options.addOption(Option.builder()
76                  .longOpt("user")
77                  .hasArg()
78                  .required()
79                  .type(String.class)
80                  .desc("Identifier of user to import as")
81                  .build());
82          options.addOption(Option.builder()
83                  .longOpt("tolerant")
84                  .desc("Don't error if a file is not valid.")
85                  .build());
86          options.addOption(Option.builder()
87                  .longOpt("allow-updates")
88                  .desc("Allow the ingest process to update existing items.")
89                  .build());
90          options.addOption(Option.builder()
91                  .longOpt("log")
92                  .hasArg()
93                  .type(String.class)
94                  .desc("Log message for action.")
95                  .build());
96          options.addOption(Option.builder()
97                  .longOpt("properties")
98                  .hasArg()
99                  .type(String.class)
100                 .desc("Provide another property file (default depends on HandlerClass)")
101                 .build());
102     }
103 
104     @Override
105     public int execWithOptions(FramedGraph<?> graph,
106             CommandLine cmdLine) throws Exception {
107 
108         GraphManager manager = GraphManagerFactory.getInstance(graph);
109 
110 
111         List<String> filePaths = Lists.newArrayList();
112         if (cmdLine.hasOption("files-from")) {
113             getPathsFromFile(cmdLine.getOptionValue("files-from"), filePaths);
114         } else if (!cmdLine.getArgList().isEmpty()) {
115             filePaths.addAll(cmdLine.getArgList());
116         } else {
117             throw new RuntimeException(getUsage());
118         }
119 
120         String logMessage = cmdLine.hasOption("log")
121                 ? cmdLine.getOptionValue("log")
122                 : "Imported from command-line";
123 
124         try {
125             // Find the agent
126             PermissionScope scope = SystemScope.getInstance();
127             if (cmdLine.hasOption("scope")) {
128                 scope = manager.getEntity(cmdLine.getOptionValue("scope"), PermissionScope.class);
129             }
130 
131             // Find the user
132             UserProfile user = manager.getEntity(cmdLine.getOptionValue("user"),
133                     UserProfile.class);
134 
135             XmlImportProperties optionalProperties = null;
136             if (cmdLine.hasOption("properties")) {
137                 XmlImportProperties properties = new XmlImportProperties(cmdLine.getOptionValue("properties"));
138                 logMessage += " Using properties file : " + cmdLine.getOptionValue("properties");
139                 optionalProperties = properties;
140             }
141 
142             ImportLog log = new SaxImportManager(graph, scope, user,
143                     cmdLine.hasOption("tolerant"),
144                     cmdLine.hasOption("allow-updates"),
145                     importer, handler,
146                     optionalProperties,
147                     Lists.<ImportCallback>newArrayList())
148                     .importFiles(filePaths, logMessage);
149             System.out.println(log);
150 
151             if (log.getErrored() > 0) {
152                 System.out.println("Errors:");
153                 for (Map.Entry<String, String> entry : log.getErrors().entrySet()) {
154                     System.out.printf(" - %-20s : %s%n", entry.getKey(),
155                             entry.getValue());
156                 }
157             }
158         } catch (RuntimeException e) {
159             e.printStackTrace();
160             return 1;
161         }
162         return 0;
163     }
164 
165     /**
166      * Read a set of file paths from an input, either a file or standard in
167      * if given the path '-'.
168      *
169      * @param listFile  A path to a local file
170      * @param filePaths An output parameter for file paths contained in
171      *                  the given file.
172      */
173     private void getPathsFromFile(String listFile, List<String> filePaths) throws Exception {
174         InputStream stream = listFile.contentEquals("-")
175                 ? System.in
176                 : Files.newInputStream(Paths.get(listFile));
177         try (BufferedReader br = new BufferedReader(new InputStreamReader(stream, Charsets.UTF_8))) {
178             String file;
179             while ((file = br.readLine()) != null) {
180                 filePaths.add(file);
181             }
182         }
183     }
184 }