1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package eu.ehri.project.commands;
21
22 import com.google.common.base.Charsets;
23 import com.google.common.collect.Lists;
24 import com.tinkerpop.frames.FramedGraph;
25 import eu.ehri.project.acl.SystemScope;
26 import eu.ehri.project.core.GraphManager;
27 import eu.ehri.project.core.GraphManagerFactory;
28 import eu.ehri.project.importers.ImportCallback;
29 import eu.ehri.project.importers.ImportLog;
30 import eu.ehri.project.importers.base.ItemImporter;
31 import eu.ehri.project.importers.base.SaxXmlHandler;
32 import eu.ehri.project.importers.managers.SaxImportManager;
33 import eu.ehri.project.importers.properties.XmlImportProperties;
34 import eu.ehri.project.models.UserProfile;
35 import eu.ehri.project.models.base.PermissionScope;
36 import org.apache.commons.cli.CommandLine;
37 import org.apache.commons.cli.Option;
38 import org.apache.commons.cli.Options;
39
40 import java.io.BufferedReader;
41 import java.io.InputStream;
42 import java.io.InputStreamReader;
43 import java.nio.file.Files;
44 import java.nio.file.Paths;
45 import java.util.List;
46 import java.util.Map;
47
48
49
50
51 public abstract class ImportCommand extends BaseCommand {
52 private final Class<? extends SaxXmlHandler> handler;
53 private final Class<? extends ItemImporter> importer;
54
55 public ImportCommand(Class<? extends SaxXmlHandler> handler, Class<? extends ItemImporter> importer) {
56 this.handler = handler;
57 this.importer = importer;
58 }
59
60 @Override
61 protected void setCustomOptions(Options options) {
62 options.addOption(Option.builder()
63 .longOpt("scope")
64 .hasArg()
65 .required()
66 .type(String.class)
67 .desc("Identifier of scope to import into, i.e. repository")
68 .build());
69 options.addOption(Option.builder("F")
70 .longOpt("files-from")
71 .hasArg()
72 .type(String.class)
73 .desc("Read list of input files from another file (or standard input, if given '-')")
74 .build());
75 options.addOption(Option.builder()
76 .longOpt("user")
77 .hasArg()
78 .required()
79 .type(String.class)
80 .desc("Identifier of user to import as")
81 .build());
82 options.addOption(Option.builder()
83 .longOpt("tolerant")
84 .desc("Don't error if a file is not valid.")
85 .build());
86 options.addOption(Option.builder()
87 .longOpt("allow-updates")
88 .desc("Allow the ingest process to update existing items.")
89 .build());
90 options.addOption(Option.builder()
91 .longOpt("log")
92 .hasArg()
93 .type(String.class)
94 .desc("Log message for action.")
95 .build());
96 options.addOption(Option.builder()
97 .longOpt("properties")
98 .hasArg()
99 .type(String.class)
100 .desc("Provide another property file (default depends on HandlerClass)")
101 .build());
102 }
103
104 @Override
105 public int execWithOptions(FramedGraph<?> graph,
106 CommandLine cmdLine) throws Exception {
107
108 GraphManager manager = GraphManagerFactory.getInstance(graph);
109
110
111 List<String> filePaths = Lists.newArrayList();
112 if (cmdLine.hasOption("files-from")) {
113 getPathsFromFile(cmdLine.getOptionValue("files-from"), filePaths);
114 } else if (!cmdLine.getArgList().isEmpty()) {
115 filePaths.addAll(cmdLine.getArgList());
116 } else {
117 throw new RuntimeException(getUsage());
118 }
119
120 String logMessage = cmdLine.hasOption("log")
121 ? cmdLine.getOptionValue("log")
122 : "Imported from command-line";
123
124 try {
125
126 PermissionScope scope = SystemScope.getInstance();
127 if (cmdLine.hasOption("scope")) {
128 scope = manager.getEntity(cmdLine.getOptionValue("scope"), PermissionScope.class);
129 }
130
131
132 UserProfile user = manager.getEntity(cmdLine.getOptionValue("user"),
133 UserProfile.class);
134
135 XmlImportProperties optionalProperties = null;
136 if (cmdLine.hasOption("properties")) {
137 XmlImportProperties properties = new XmlImportProperties(cmdLine.getOptionValue("properties"));
138 logMessage += " Using properties file : " + cmdLine.getOptionValue("properties");
139 optionalProperties = properties;
140 }
141
142 ImportLog log = new SaxImportManager(graph, scope, user,
143 cmdLine.hasOption("tolerant"),
144 cmdLine.hasOption("allow-updates"),
145 importer, handler,
146 optionalProperties,
147 Lists.<ImportCallback>newArrayList())
148 .importFiles(filePaths, logMessage);
149 System.out.println(log);
150
151 if (log.getErrored() > 0) {
152 System.out.println("Errors:");
153 for (Map.Entry<String, String> entry : log.getErrors().entrySet()) {
154 System.out.printf(" - %-20s : %s%n", entry.getKey(),
155 entry.getValue());
156 }
157 }
158 } catch (RuntimeException e) {
159 e.printStackTrace();
160 return 1;
161 }
162 return 0;
163 }
164
165
166
167
168
169
170
171
172
173 private void getPathsFromFile(String listFile, List<String> filePaths) throws Exception {
174 InputStream stream = listFile.contentEquals("-")
175 ? System.in
176 : Files.newInputStream(Paths.get(listFile));
177 try (BufferedReader br = new BufferedReader(new InputStreamReader(stream, Charsets.UTF_8))) {
178 String file;
179 while ((file = br.readLine()) != null) {
180 filePaths.add(file);
181 }
182 }
183 }
184 }