View Javadoc

1   /*
2    * Copyright 2015 Data Archiving and Networked Services (an institute of
3    * Koninklijke Nederlandse Akademie van Wetenschappen), King's College London,
4    * Georg-August-Universitaet Goettingen Stiftung Oeffentlichen Rechts
5    *
6    * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by
7    * the European Commission - subsequent versions of the EUPL (the "Licence");
8    * You may not use this work except in compliance with the Licence.
9    * You may obtain a copy of the Licence at:
10   *
11   * https://joinup.ec.europa.eu/software/page/eupl
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the Licence is distributed on an "AS IS" basis,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the Licence for the specific language governing
17   * permissions and limitations under the Licence.
18   */
19  
20  package eu.ehri.project.commands;
21  
22  import com.tinkerpop.blueprints.Vertex;
23  import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
24  import com.tinkerpop.blueprints.util.io.graphson.GraphSONReader;
25  import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
26  import com.tinkerpop.frames.FramedGraph;
27  import eu.ehri.project.core.impl.Neo4jGraphManager;
28  import eu.ehri.project.core.impl.neo4j.Neo4j2Graph;
29  import org.apache.commons.cli.CommandLine;
30  import org.apache.commons.cli.Option;
31  import org.apache.commons.cli.Options;
32  
33  import java.io.IOException;
34  import java.io.InputStream;
35  import java.io.OutputStream;
36  import java.nio.file.Files;
37  import java.nio.file.Paths;
38  import java.util.zip.GZIPInputStream;
39  
40  /**
41   * Dump the complete graph as graphSON file, or import such a dump
42   * <p>
43   * Example usage:
44   * <pre>
45   *     <code>
46   * # stop the server
47   * $NEO4J_HOME/bin/neo4j stop
48   * # save a dump
49   * ./scripts/cmd graphson -d out graph.json
50   * # or
51   * ./scripts/cmd graphson -d out - &gt; graph.json
52   * # edit it
53   * # remove the graph
54   * rm -rf $NEO4J_HOME/data/graph.db
55   * # load edited graph
56   * ./scripts/cmd graphson -d in graph.json
57   * # start server
58   * $NEO4J_HOME/bin/neo4j start
59   *     </code>
60   * </pre>
61   */
62  public class GraphSON extends BaseCommand {
63  
64      final static String NAME = "graphson";
65  
66      @Override
67      public String getHelp() {
68          return "Load or dump GraphSON data.";
69      }
70  
71      @Override
72      public String getHelpFooter() {
73          return "Default is to dump to stdout";
74      }
75  
76      @Override
77      public String getUsage() {
78          return String.format("%s [OPTIONS] [--load <filename>|--dump <filename>]", NAME);
79      }
80  
81      @Override
82      protected void setCustomOptions(Options options) {
83          options.addOption(Option.builder("l")
84                  .hasArg().type(String.class)
85                  .longOpt("load")
86                  .desc("Load a dump file").build());
87          options.addOption(Option.builder("d")
88                  .hasArg().type(String.class)
89                  .longOpt("dump")
90                  .desc("Save a dump file").build());
91          options.addOption(Option.builder("b")
92                  .hasArg().type(Integer.class)
93                  .longOpt("buffer-size")
94                  .desc("Transaction buffer size").build());
95          options.addOption(Option.builder()
96                  .longOpt("skip-setting-labels")
97                  .desc("Initialize indices after load").build());
98      }
99  
100     @Override
101     public int execWithOptions(FramedGraph<?> graph,
102             CommandLine cmdLine) throws Exception {
103 
104         // check if option is useful, otherwise print the help and bail out
105         if (cmdLine.hasOption("dump")) {
106             saveDump(graph, cmdLine.getOptionValue("dump"), cmdLine);
107         } else if (cmdLine.hasOption("load")) {
108             loadDump(graph, cmdLine.getOptionValue("load"), cmdLine);
109         } else {
110             saveDump(graph, "-", cmdLine);
111         }
112 
113         return 0;
114     }
115 
116     private void saveDump(FramedGraph<?> graph,
117             String filePath, CommandLine cmdLine) throws IOException {
118 
119         // if the file is '-' that means we do standard out
120         if (filePath.contentEquals("-")) {
121             // to stdout
122             GraphSONWriter.outputGraph(graph, System.out, GraphSONMode.EXTENDED);
123         } else {
124             // try to open or create the file for writing
125             OutputStream out = Files.newOutputStream(Paths.get(filePath));
126             GraphSONWriter.outputGraph(graph, out, GraphSONMode.EXTENDED);
127             out.close();
128         }
129     }
130 
131     private void loadDump(FramedGraph<?> graph,
132             String filePath, CommandLine cmdLine) throws Exception {
133         GraphSONReader reader = new GraphSONReader(graph);
134 
135         InputStream readStream = System.in;
136         if (!filePath.equals("-")) {
137             InputStream inputStream = Files.newInputStream(Paths.get(filePath));
138             readStream = filePath.toLowerCase().endsWith(".gz")
139                     ? new GZIPInputStream(inputStream)
140                     : inputStream;
141         }
142 
143         int bufferSize = cmdLine.hasOption("buffer-size")
144                 ? Integer.parseInt(cmdLine.getOptionValue("buffer-size"))
145                 : 1000;
146 
147         try {
148             reader.inputGraph(readStream, bufferSize);
149             if (!cmdLine.hasOption("skip-setting-labels")) {
150                 if (graph.getBaseGraph() instanceof Neo4j2Graph) {
151                     // safe, due to the above instanceof
152                     @SuppressWarnings("unchecked")
153                     FramedGraph<Neo4j2Graph> neo4j2Graph = ((FramedGraph<Neo4j2Graph>) graph);
154                     Neo4jGraphManager manager = new Neo4jGraphManager<>(neo4j2Graph);
155                     int i = 0;
156                     for (Vertex v : graph.getVertices()) {
157                         manager.setLabels(v);
158                         i++;
159                         if (i % 10000 == 0) {
160                             neo4j2Graph.getBaseGraph().commit();
161                         }
162                     }
163                     System.err.println("Labelled " + i + " vertices");
164                 }
165             }
166         } finally {
167             readStream.close();
168         }
169     }
170 }