View Javadoc

1   /*
2    * Copyright 2015 Data Archiving and Networked Services (an institute of
3    * Koninklijke Nederlandse Akademie van Wetenschappen), King's College London,
4    * Georg-August-Universitaet Goettingen Stiftung Oeffentlichen Rechts
5    *
6    * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by
7    * the European Commission - subsequent versions of the EUPL (the "Licence");
8    * You may not use this work except in compliance with the Licence.
9    * You may obtain a copy of the Licence at:
10   *
11   * https://joinup.ec.europa.eu/software/page/eupl
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the Licence is distributed on an "AS IS" basis,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the Licence for the specific language governing
17   * permissions and limitations under the Licence.
18   */
19  
20  package eu.ehri.project.importers.managers;
21  
22  import com.fasterxml.jackson.databind.MappingIterator;
23  import com.fasterxml.jackson.databind.ObjectReader;
24  import com.fasterxml.jackson.dataformat.csv.CsvMapper;
25  import com.fasterxml.jackson.dataformat.csv.CsvSchema;
26  import com.google.common.base.Charsets;
27  import com.google.common.collect.Maps;
28  import com.tinkerpop.frames.FramedGraph;
29  import eu.ehri.project.exceptions.ValidationError;
30  import eu.ehri.project.importers.ImportLog;
31  import eu.ehri.project.importers.base.ItemImporter;
32  import eu.ehri.project.importers.exceptions.InputParseError;
33  import eu.ehri.project.importers.util.ImportHelpers;
34  import eu.ehri.project.models.base.Actioner;
35  import eu.ehri.project.models.base.PermissionScope;
36  import eu.ehri.project.persistence.ActionManager;
37  import org.slf4j.Logger;
38  import org.slf4j.LoggerFactory;
39  
40  import java.io.IOException;
41  import java.io.InputStream;
42  import java.io.InputStreamReader;
43  import java.lang.reflect.InvocationTargetException;
44  import java.util.Map;
45  
46  /**
47   * Import manager to use with CSV files.
48   * When used to import DocumentaryUnits, make sure to have a 'sourceFileId' column as well.
49   */
50  public class CsvImportManager extends AbstractImportManager {
51  
52      private static final Character VALUE_DELIMITER = ';';
53  
54      private static final Logger logger = LoggerFactory.getLogger(CsvImportManager.class);
55  
56      public CsvImportManager(FramedGraph<?> framedGraph,
57              PermissionScope permissionScope, Actioner actioner,
58              boolean tolerant,
59              boolean allowUpdates, Class<? extends ItemImporter> importerClass) {
60          super(framedGraph, permissionScope, actioner, tolerant, allowUpdates, importerClass);
61      }
62  
63      /**
64       * Import CSV from the given InputStream, as part of the given action.
65       *
66       * @param stream  the input stream
67       * @param context the event context in which the ingest is happening
68       * @param log     an import log instance
69       */
70      @Override
71      protected void importInputStream(InputStream stream, String tag, final ActionManager.EventContext context,
72              final ImportLog log) throws IOException, ValidationError, InputParseError {
73  
74          try {
75              ItemImporter importer = importerClass
76                      .getConstructor(FramedGraph.class, PermissionScope.class, Actioner.class, ImportLog.class)
77                      .newInstance(framedGraph, permissionScope, actioner, log);
78              logger.debug("importer of class " + importer.getClass());
79  
80              importer.addCallback(mutation -> defaultImportCallback(log, context, mutation));
81              importer.addErrorCallback(ex -> defaultErrorCallback(log, ex));
82  
83              CsvSchema schema = CsvSchema.emptySchema().withColumnSeparator(VALUE_DELIMITER).withHeader();
84              ObjectReader reader = new CsvMapper().readerFor(Map.class).with(schema);
85  
86              try (InputStreamReader s = new InputStreamReader(stream, Charsets.UTF_8);
87                   MappingIterator<Map<String, String>> valueIterator = reader.readValues(s)) {
88                  while (valueIterator.hasNext()) {
89                      Map<String, String> rawData = valueIterator.next();
90                      Map<String, Object> dataMap = Maps.newHashMap();
91                      for (Map.Entry<String, String> entry : rawData.entrySet()) {
92                          ImportHelpers.putPropertyInGraph(dataMap,
93                                  entry.getKey().replaceAll("\\s", ""), entry.getValue());
94                      }
95                      try {
96                          importer.importItem(dataMap);
97                      } catch (ValidationError e) {
98                          if (isTolerant()) {
99                              logger.error("Validation error importing item: {}", e);
100                         } else {
101                             throw e;
102                         }
103                     }
104                 }
105             }
106         } catch (IllegalAccessException | InvocationTargetException |
107                 InstantiationException | NoSuchMethodException e) {
108             throw new RuntimeException(e);
109         }
110     }
111 }