diff --git a/.travis.yml b/.travis.yml index c1593ba9848e..57344f9e4004 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ os: matrix: allow_failures: - os: windows - - os: osx cache: directories: diff --git a/LICENSE b/LICENSE index d00ac390964b..261eeb9e9f8b 100644 --- a/LICENSE +++ b/LICENSE @@ -199,22 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - ======================================================================= - - Gradoop subcomponents: - - The gradoop project contains subcomponents in the source code - release with separate copyright notices and license terms. Your use of - the source code for these subcomponents is subject to the terms and - conditions of their respective licenses. - - - ----------------------------------------------------------------------- - The Apache License 2.0 - ----------------------------------------------------------------------- - - - Flink 1.5.0 - - Hbase 1.3.1 - - Accumulo 1.9.0 - - MongoDB 3.6.3 \ No newline at end of file diff --git a/NOTICE b/NOTICE index c2960033d369..ce30acd912d5 100644 --- a/NOTICE +++ b/NOTICE @@ -4,10 +4,25 @@ Copyright 2014-2019 Leipzig University (Database Research Group) This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -======================================================================= +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) -Gradoop contains subcomponents with separate copyright notices and -license terms. Your use of the source code for these subcomponents -is subject to the terms and conditions of their respective licenses. +- com.github.s1ck:gdl:0.3 +- commons-cli:commons-cli:1.4 +- log4j:log4j:1.2.17 +- me.lemire.integercompression:JavaFastPFOR:0.1.10 +- org.apache.accumulo:accumulo-core:1.9.0 +- org.apache.flink:flink-clients_2.11:1.7.2 +- org.apache.flink:flink-core:1.7.2 +- org.apache.flink:flink-gelly_2.11:1.7.2 +- org.apache.flink:flink-hadoop-compatibility_2.11:1.7.2 +- org.apache.flink:flink-hbase_2.11:1.7.2 +- org.apache.flink:flink-java:1.7.2 +- org.apache.flink:flink-shaded-hadoop2:1.7.2 +- org.apache.hbase:hbase-hadoop-compat:1.4.3 +- org.apache.hbase:hbase-server:1.4.3 +- org.codehaus.jettison:jettison:1.3.7 -See the LICENSE file for a list of subcomponents respective licenses. \ No newline at end of file +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.esotericsoftware:kryo:4.0.2 diff --git a/NOTICE-binary b/NOTICE-binary new file mode 100644 index 000000000000..be485339d866 --- /dev/null +++ b/NOTICE-binary @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Gradoop +// ------------------------------------------------------------------ + +gradoop-accumulo +Copyright 2014-2019 Leipzig University (Database Research Group) + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.github.s1ck:gdl:0.3 +- com.google.guava:guava +- commons-cli:commons-cli:1.4 +- log4j:log4j:1.2.17 +- me.lemire.integercompression:JavaFastPFOR:0.1.10 +- org.codehaus.jettison:jettison:1.3.7 +- org.objenesis:objenesis:2.5.1 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.esotericsoftware:kryo:4.0.2 +- com.esotericsoftware:minlog:1.3.0 +- org.antlr:antlr4-runtime diff --git a/README.md b/README.md index 2036833e593c..3eec25e29977 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Apache License, Version 2.0, January 2004](https://img.shields.io/github/license/apache/maven.svg?label=License)](https://www.apache.org/licenses/LICENSE-2.0) -[![Maven Central](https://img.shields.io/badge/Maven_Central-0.4.4-blue.svg?label=Maven%20Central)](http://search.maven.org/#search%7Cga%7C1%7Cgradoop) +[![Maven Central](https://img.shields.io/badge/Maven_Central-0.4.5-blue.svg?label=Maven%20Central)](http://search.maven.org/#search%7Cga%7C1%7Cgradoop) [![Build Status](https://travis-ci.org/dbs-leipzig/gradoop.svg?branch=master)](https://travis-ci.org/dbs-leipzig/gradoop) [![Code Quality: Java](https://img.shields.io/lgtm/grade/java/g/dbs-leipzig/gradoop.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/dbs-leipzig/gradoop/context:java) [![Total Alerts](https://img.shields.io/lgtm/alerts/g/dbs-leipzig/gradoop.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/dbs-leipzig/gradoop/alerts) @@ -111,7 +111,7 @@ Stable: org.gradoop gradoop-flink - 0.4.4 + 0.4.5 ``` @@ -134,17 +134,17 @@ Latest nightly build (additional repository is required): ``` -In any case you also need Apache Flink (version 1.6.0): +In any case you also need Apache Flink (version 1.7.2): ``` org.apache.flink flink-java - 1.7.0 + 1.7.2 org.apache.flink flink-clients_2.11 - 1.7.0 + 1.7.2 ``` diff --git a/gradoop-checkstyle/pom.xml b/gradoop-checkstyle/pom.xml index 9eb6eaa1452a..e07d2a165960 100644 --- a/gradoop-checkstyle/pom.xml +++ b/gradoop-checkstyle/pom.xml @@ -5,7 +5,7 @@ org.gradoop gradoop-parent - 0.4.4 + 0.4.5 gradoop-checkstyle diff --git a/gradoop-checkstyle/src/main/resources/gradoop/checkstyle-suppressions.xml b/gradoop-checkstyle/src/main/resources/gradoop/checkstyle-suppressions.xml index f81cae98c4a1..7b78f139aca3 100644 --- a/gradoop-checkstyle/src/main/resources/gradoop/checkstyle-suppressions.xml +++ b/gradoop-checkstyle/src/main/resources/gradoop/checkstyle-suppressions.xml @@ -14,6 +14,9 @@ + - \ No newline at end of file + diff --git a/gradoop-common/pom.xml b/gradoop-common/pom.xml index 2f623e5e91f7..7514cc1bdcaf 100644 --- a/gradoop-common/pom.xml +++ b/gradoop-common/pom.xml @@ -1,11 +1,12 @@ - + 4.0.0 org.gradoop gradoop-parent - 0.4.4 + 0.4.5 gradoop-common diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/api/strategies/PropertyValueStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/api/strategies/PropertyValueStrategy.java new file mode 100644 index 000000000000..ac250810d703 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/api/strategies/PropertyValueStrategy.java @@ -0,0 +1,104 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.api.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.io.IOException; + +/** + * Interface defining the methods necessary to handle the (de-) serialization of a + * {@link PropertyValue}'s value. + * + * @param The property value type. + */ +public interface PropertyValueStrategy { + + /** + * Writes the given value to the provided {@link DataOutputView}. + * The argument {@code value} can not be {@code null}. + * + * @param value to be written to the {@link DataOutputView}. + * @param outputView that the value is written to. + * @throws IOException if write process fails. + */ + void write(T value, DataOutputView outputView) throws IOException; + + /** + * Reads raw bytes from the given {@link DataInputView} and deserializes the contained object. + * + * @param inputView containing serialized object. + * @param typeByte byte needed to indicate whether serialized object has a variable length. + * @return deserialized object. + * @throws IOException when reading or deserialization of the object fails. + */ + T read(DataInputView inputView, byte typeByte) throws IOException; + + /** + * Compares two objects. + * + * @param value first object. + * @param other second object. + * @return a negative integer, zero, or a positive integer as first object is less than, equal to, + * or greater than the second object. + * @throws IllegalArgumentException when {@code other} is not comparable to {@code value}. + */ + int compare(T value, Object other); + + /** + * Checks if given object is an instance of the data type the specific strategy handles. + * + * @param value to be checked. + * @return true if {@code value} is an instance of the data type this strategy handles. + * False otherwise. + */ + boolean is(Object value); + + /** + * Gets the class of the data type the specific strategy handles. + * + * @return class of the type handled by this strategy. + */ + Class getType(); + + /** + * Deserializes an object from the provided byte array. + * + * @param bytes representing a serialized object. + * @return an instance of the type handled by this strategy. + * @throws IOException on failure + */ + T get(byte[] bytes) throws IOException; + + /** + * Gets a byte which represents the data type the specific strategy handles. + * + * @return a byte. + */ + byte getRawType(); + + /** + * Serializes the given object. + * The argument {@code value} can not be {@code null}. + * + * @param value the object to be serialized. + * @return byte array representation of the provided object. + * @throws IOException on failure + */ + byte[] getRawBytes(T value) throws IOException; +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/package-info.java b/gradoop-common/src/main/java/org/gradoop/common/model/api/strategies/package-info.java similarity index 84% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/package-info.java rename to gradoop-common/src/main/java/org/gradoop/common/model/api/strategies/package-info.java index 3bf995e61773..f7dffe619a17 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/package-info.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/api/strategies/package-info.java @@ -14,6 +14,6 @@ * limitations under the License. */ /** - * Contains sampling statistics functions + * Contains interfaces that are related to property value strategies. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics.functions; +package org.gradoop.common.model.api.strategies; diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/id/GradoopIdSet.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/id/GradoopIdSet.java index 325b2031034e..a2c9aff81f7f 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/id/GradoopIdSet.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/id/GradoopIdSet.java @@ -265,9 +265,6 @@ public boolean isEmpty() { return ids.isEmpty(); } - /** - * {@inheritDoc} - */ @Override public Iterator iterator() { return ids.iterator(); diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/Edge.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/Edge.java index 3072eb1cd0b9..2c5b5e6dd3f6 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/Edge.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/Edge.java @@ -59,43 +59,26 @@ public Edge(final GradoopId id, final String label, final GradoopId sourceId, this.targetId = targetId; } - /** - * {@inheritDoc} - */ @Override public GradoopId getSourceId() { return sourceId; } - /** - * {@inheritDoc} - * @param sourceId - */ @Override public void setSourceId(GradoopId sourceId) { this.sourceId = sourceId; } - /** - * {@inheritDoc} - */ @Override public GradoopId getTargetId() { return targetId; } - /** - * {@inheritDoc} - * @param targetId - */ @Override public void setTargetId(GradoopId targetId) { this.targetId = targetId; } - /** - * {@inheritDoc} - */ @Override public String toString() { return String.format("(%s)-[%s]->(%s)", diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/EdgeFactory.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/EdgeFactory.java index 4aa42c0d7913..f6c7bd6d3f5b 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/EdgeFactory.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/EdgeFactory.java @@ -35,18 +35,12 @@ public class EdgeFactory implements EPGMEdgeFactory, Serializable { */ private static final long serialVersionUID = 42L; - /** - * {@inheritDoc} - */ @Override public Edge createEdge(GradoopId sourceVertexId, GradoopId targetVertexId) { return initEdge(GradoopId.get(), sourceVertexId, targetVertexId); } - /** - * {@inheritDoc} - */ @Override public Edge initEdge(final GradoopId id, final GradoopId sourceVertexId, final GradoopId targetVertexId) { @@ -54,27 +48,18 @@ public Edge initEdge(final GradoopId id, final GradoopId sourceVertexId, targetVertexId); } - /** - * {@inheritDoc} - */ @Override public Edge createEdge(String label, GradoopId sourceVertexId, GradoopId targetVertexId) { return initEdge(GradoopId.get(), label, sourceVertexId, targetVertexId); } - /** - * {@inheritDoc} - */ @Override public Edge initEdge(final GradoopId id, final String label, final GradoopId sourceVertexId, final GradoopId targetVertexId) { return initEdge(id, label, sourceVertexId, targetVertexId, null, null); } - /** - * {@inheritDoc} - */ @Override public Edge createEdge(String label, GradoopId sourceVertexId, GradoopId targetVertexId, Properties properties) { @@ -82,9 +67,6 @@ public Edge createEdge(String label, GradoopId sourceVertexId, label, sourceVertexId, targetVertexId, properties); } - /** - * {@inheritDoc} - */ @Override public Edge initEdge( GradoopId id, @@ -97,9 +79,6 @@ public Edge initEdge( initEdge(id, label, sourceVertexId, targetVertexId, properties, null); } - /** - * {@inheritDoc} - */ @Override public Edge createEdge(String label, GradoopId sourceVertexId, GradoopId targetVertexId, GradoopIdSet graphIds) { @@ -107,9 +86,6 @@ public Edge createEdge(String label, GradoopId sourceVertexId, label, sourceVertexId, targetVertexId, graphIds); } - /** - * {@inheritDoc} - */ @Override public Edge initEdge(final GradoopId id, final String label, final GradoopId sourceVertexId, final GradoopId targetVertexId, @@ -117,9 +93,6 @@ public Edge initEdge(final GradoopId id, final String label, return initEdge(id, label, sourceVertexId, targetVertexId, null, graphs); } - /** - * {@inheritDoc} - */ @Override public Edge createEdge(String label, GradoopId sourceVertexId, GradoopId targetVertexId, Properties properties, @@ -128,9 +101,6 @@ public Edge createEdge(String label, GradoopId sourceVertexId, label, sourceVertexId, targetVertexId, properties, graphIds); } - /** - * {@inheritDoc} - */ @Override public Edge initEdge(final GradoopId id, final String label, final GradoopId sourceVertexId, final GradoopId targetVertexId, diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/Element.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/Element.java index 00f156a1e5f5..1553087a5853 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/Element.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/Element.java @@ -64,75 +64,48 @@ protected Element( this.properties = properties; } - /** - * {@inheritDoc} - */ @Override public GradoopId getId() { return id; } - /** - * {@inheritDoc} - */ @Override public void setId(GradoopId id) { this.id = id; } - /** - * {@inheritDoc} - */ @Override public String getLabel() { return label; } - /** - * {@inheritDoc} - */ @Override public void setLabel(String label) { this.label = label; } - /** - * {@inheritDoc} - */ @Override @Nullable public Properties getProperties() { return properties; } - /** - * {@inheritDoc} - */ @Override public Iterable getPropertyKeys() { return (properties != null) ? properties.getKeys() : null; } - /** - * {@inheritDoc} - */ @Override public PropertyValue getPropertyValue(String key) { // TODO: return PropertyValue.NULL_VALUE instead? return (properties != null) ? properties.get(key) : null; } - /** - * {@inheritDoc} - */ @Override public void setProperties(Properties properties) { this.properties = properties; } - /** - * {@inheritDoc} - */ @Override public void setProperty(Property property) { Preconditions.checkNotNull(property, "Property was null"); @@ -140,27 +113,18 @@ public void setProperty(Property property) { this.properties.set(property); } - /** - * {@inheritDoc} - */ @Override public void setProperty(String key, Object value) { initProperties(); this.properties.set(key, value); } - /** - * {@inheritDoc} - */ @Override public void setProperty(String key, PropertyValue value) { initProperties(); this.properties.set(key, value); } - /** - * {@inheritDoc} - */ @Override public PropertyValue removeProperty(String key) { return this.properties != null ? properties.remove(key) : null; @@ -171,17 +135,11 @@ public int getPropertyCount() { return (this.properties != null) ? this.properties.size() : 0; } - /** - * {@inheritDoc} - */ @Override public boolean hasProperty(String key) { return this.properties != null && this.properties.containsKey(key); } - /** - * {@inheritDoc} - */ @Override public boolean equals(Object o) { if (this == o) { @@ -196,9 +154,6 @@ public boolean equals(Object o) { return !(id != null ? !id.equals(that.id) : that.id != null); } - /** - * {@inheritDoc} - */ @Override public int hashCode() { int result = id.hashCode(); diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/GraphElement.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/GraphElement.java index e0f41e904839..6fb975dffa07 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/GraphElement.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/GraphElement.java @@ -52,17 +52,11 @@ protected GraphElement(GradoopId id, String label, this.graphIds = graphIds; } - /** - * {@inheritDoc} - */ @Override public GradoopIdSet getGraphIds() { return graphIds; } - /** - * {@inheritDoc} - */ @Override public void addGraphId(GradoopId graphId) { if (graphIds == null) { @@ -71,17 +65,11 @@ public void addGraphId(GradoopId graphId) { graphIds.add(graphId); } - /** - * {@inheritDoc} - */ @Override public void setGraphIds(GradoopIdSet graphIds) { this.graphIds = graphIds; } - /** - * {@inheritDoc} - */ @Override public void resetGraphIds() { if (graphIds != null) { @@ -89,17 +77,11 @@ public void resetGraphIds() { } } - /** - * {@inheritDoc} - */ @Override public int getGraphCount() { return (graphIds != null) ? graphIds.size() : 0; } - /** - * {@inheritDoc} - */ @Override public String toString() { return String.format("%s @ %s", super.toString(), graphIds); diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/GraphHeadFactory.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/GraphHeadFactory.java index 38402c039898..10302ebccf9a 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/GraphHeadFactory.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/GraphHeadFactory.java @@ -34,49 +34,31 @@ public class GraphHeadFactory implements EPGMGraphHeadFactory, */ private static final long serialVersionUID = 42L; - /** - * {@inheritDoc} - */ @Override public GraphHead createGraphHead() { return initGraphHead(GradoopId.get()); } - /** - * {@inheritDoc} - */ @Override public GraphHead initGraphHead(final GradoopId id) { return initGraphHead(id, GradoopConstants.DEFAULT_GRAPH_LABEL, null); } - /** - * {@inheritDoc} - */ @Override public GraphHead createGraphHead(String label) { return initGraphHead(GradoopId.get(), label); } - /** - * {@inheritDoc} - */ @Override public GraphHead initGraphHead(final GradoopId id, final String label) { return initGraphHead(id, label, null); } - /** - * {@inheritDoc} - */ @Override public GraphHead createGraphHead(String label, Properties properties) { return initGraphHead(GradoopId.get(), label, properties); } - /** - * {@inheritDoc} - */ @Override public GraphHead initGraphHead(final GradoopId id, final String label, Properties properties) { diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/VertexFactory.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/VertexFactory.java index 6b2997bc3ab2..c693012a9f40 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/VertexFactory.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/pojo/VertexFactory.java @@ -34,84 +34,54 @@ public class VertexFactory implements EPGMVertexFactory, Serializable { */ private static final long serialVersionUID = 42L; - /** - * {@inheritDoc} - */ @Override public Vertex createVertex() { return initVertex(GradoopId.get()); } - /** - * {@inheritDoc} - */ @Override public Vertex initVertex(final GradoopId vertexID) { return initVertex(vertexID, GradoopConstants.DEFAULT_VERTEX_LABEL, null, null); } - /** - * {@inheritDoc} - */ @Override public Vertex createVertex(String label) { return initVertex(GradoopId.get(), label); } - /** - * {@inheritDoc} - */ @Override public Vertex initVertex(final GradoopId vertexID, final String label) { return initVertex(vertexID, label, null, null); } - /** - * {@inheritDoc} - */ @Override public Vertex createVertex(String label, Properties properties) { return initVertex(GradoopId.get(), label, properties); } - /** - * {@inheritDoc} - */ @Override public Vertex initVertex(final GradoopId vertexID, final String label, Properties properties) { return initVertex(vertexID, label, properties, null); } - /** - * {@inheritDoc} - */ @Override public Vertex createVertex(String label, GradoopIdSet graphIds) { return initVertex(GradoopId.get(), label, graphIds); } - /** - * {@inheritDoc} - */ @Override public Vertex initVertex(final GradoopId vertexID, final String label, final GradoopIdSet graphs) { return initVertex(vertexID, label, null, graphs); } - /** - * {@inheritDoc} - */ @Override public Vertex createVertex(String label, Properties properties, GradoopIdSet graphIds) { return initVertex(GradoopId.get(), label, properties, graphIds); } - /** - * {@inheritDoc} - */ @Override public Vertex initVertex(final GradoopId id, final String label, final Properties properties, final GradoopIdSet graphs) { diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/Properties.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/Properties.java index 7575d6121daa..0dbfa5f37a5e 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/Properties.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/Properties.java @@ -258,9 +258,6 @@ public int hashCode() { return properties != null ? properties.hashCode() : 0; } - /** - * {@inheritDoc} - */ @Override public Iterator iterator() { return toList().iterator(); @@ -277,9 +274,6 @@ public List toList() { .collect(Collectors.toList()); } - /** - * {@inheritDoc} - */ @Override public void write(DataOutputView outputView) throws IOException { outputView.writeInt(properties.size()); @@ -290,10 +284,6 @@ public void write(DataOutputView outputView) throws IOException { } } - - /** - * {@inheritDoc} - */ @Override public void read(DataInputView inputView) throws IOException { int propertyCount = inputView.readInt(); @@ -310,9 +300,6 @@ public void read(DataInputView inputView) throws IOException { } } - /** - * {@inheritDoc} - */ @Override public String toString() { return toList().stream() diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/PropertyValue.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/PropertyValue.java index 0ed7a2be073f..67d9250a1155 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/PropertyValue.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/PropertyValue.java @@ -15,21 +15,14 @@ */ package org.gradoop.common.model.impl.properties; -import edu.umd.cs.findbugs.annotations.SuppressWarnings; import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.types.Value; -import org.apache.hadoop.hbase.util.Bytes; -import org.gradoop.common.model.impl.id.GradoopId; import org.gradoop.common.exceptions.UnsupportedTypeException; +import org.gradoop.common.model.api.strategies.PropertyValueStrategy; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.properties.strategies.PropertyValueStrategyFactory; import org.gradoop.common.util.GradoopConstants; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; import java.io.IOException; import java.io.Serializable; import java.math.BigDecimal; @@ -37,12 +30,9 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.util.Arrays; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; /** @@ -120,7 +110,6 @@ public class PropertyValue implements Value, Serializable, Comparable} for {@link java.util.Set} */ public static final transient byte TYPE_SET = 0x0f; - /** * Value offset in byte */ @@ -132,7 +121,7 @@ public class PropertyValue implements Value, Serializable, Comparable TYPE_MAPPING = getTypeMap(); - - /** - * Stores the type and the value - */ - private byte[] rawBytes; + private Object value; /** * Default constructor. @@ -196,26 +180,47 @@ public static PropertyValue create(Object value) { return new PropertyValue(value); } + /** + * Create a {@link PropertyValue} that wraps a byte array. + * + * @param rawBytes array to wrap + * @return new instance of {@link PropertyValue} + */ + public static PropertyValue fromRawBytes(byte[] rawBytes) { + return new PropertyValue(rawBytes); + } + /** * Creates a deep copy of the property value. * * @return property value */ public PropertyValue copy() { - return create(getObject()); + return new PropertyValue(getRawBytes()); } //---------------------------------------------------------------------------- // Type checking //---------------------------------------------------------------------------- + /** + * Check if the property value type is an instance of a certain class. + * + * @param clazz class to check against + * @return true if the attribute {@code value} is an object of the provided class, false + * otherwise + */ + public boolean is(Class clazz) { + return PropertyValueStrategyFactory.get(clazz).is(value); + } + /** * True, if the value represents {@code null}. * * @return true, if {@code null} value */ public boolean isNull() { - return rawBytes[0] == TYPE_NULL; + return getRawBytes()[0] == PropertyValue.TYPE_NULL; } /** @@ -224,126 +229,140 @@ public boolean isNull() { * @return true, if {@code boolean} value */ public boolean isBoolean() { - return rawBytes[0] == TYPE_BOOLEAN; + return is(Boolean.class); } + /** * True, if the wrapped value is of type {@code short}. * * @return true, if {@code short} value */ public boolean isShort() { - return rawBytes[0] == TYPE_SHORT; + return is(Short.class); } + /** * True, if the wrapped value is of type {@code int}. * * @return true, if {@code int} value */ public boolean isInt() { - return rawBytes[0] == TYPE_INTEGER; + return is(Integer.class); } + /** * True, if the wrapped value is of type {@code long}. * * @return true, if {@code long} value */ public boolean isLong() { - return rawBytes[0] == TYPE_LONG; + return is(Long.class); } + /** * True, if the wrapped value is of type {@code float}. * * @return true, if {@code float} value */ public boolean isFloat() { - return rawBytes[0] == TYPE_FLOAT; + return is(Float.class); } + /** * True, if the wrapped value is of type {@code double}. * * @return true, if {@code double} value */ public boolean isDouble() { - return rawBytes[0] == TYPE_DOUBLE; + return is(Double.class); } + /** - * True, if the wrapped value is of type {@code String}. + * True, if the wrapped value is of type {@link String}. * - * @return true, if {@code String} value + * @return true, if {@link String} value */ public boolean isString() { - return rawBytes[0] == TYPE_STRING; + return is(String.class); } + /** - * True, if the wrapped value is of type {@code BigDecimal}. + * True, if the wrapped value is of type {@link BigDecimal}. * - * @return true, if {@code BigDecimal} value + * @return true, if {@link BigDecimal} value * @see BigDecimal */ public boolean isBigDecimal() { - return rawBytes[0] == TYPE_BIG_DECIMAL; + return is(BigDecimal.class); } + /** - * True, if the wrapped value is of type {@code GradoopId}. + * True, if the wrapped value is of type {@link GradoopId}. * - * @return true, if {@code GradoopId} value + * @return true, if {@link GradoopId} value */ public boolean isGradoopId() { - return rawBytes[0] == TYPE_GRADOOP_ID; + return is(GradoopId.class); } + /** - * True, if the wrapped value is of type {@code Map}. + * True, if the wrapped value is of type {@link Map}. * - * @return true, if {@code Map} value + * @return true, if {@link Map} value */ public boolean isMap() { - return rawBytes[0] == TYPE_MAP; + return is(Map.class); } + /** - * True, if the wrapped value is of type {@code List}. + * True, if the wrapped value is of type {@link List}. * - * @return true, if {@code List} value + * @return true, if {@link List} value */ public boolean isList() { - return rawBytes[0] == TYPE_LIST; + return is(List.class); } + /** - * True, if the wrapped value is of type {@code LocalDate}. + * True, if the wrapped value is of type {@link LocalDate}. * - * @return true, if {@code LocalDate} value + * @return true, if {@link LocalDate} value */ public boolean isDate() { - return rawBytes[0] == TYPE_DATE; + return is(LocalDate.class); } + /** - * True, if the wrapped value is of type {@code LocalTime}. + * True, if the wrapped value is of type {@link LocalTime}. * - * @return true, if {@code LocalTime} value + * @return true, if {@link LocalTime} value */ public boolean isTime() { - return rawBytes[0] == TYPE_TIME; + return is(LocalTime.class); } + /** - * True, if the wrapped value is of type {@code LocalDateTime}. + * True, if the wrapped value is of type {@link LocalDateTime}. * - * @return true, if {@code LocalDateTime} value + * @return true, if {@link LocalDateTime} value */ public boolean isDateTime() { - return rawBytes[0] == TYPE_DATETIME; + return is(LocalDateTime.class); } + /** - * True, if the wrapped value is of type {@code Set}. + * True, if the wrapped value is of type {@link Set}. * - * @return true, if {@code Set} value + * @return true, if {@link Set} value */ public boolean isSet() { - return rawBytes[0] == TYPE_SET; + return is(Set.class); } /** - * True, if the wrapped value is a subtype of {@code Number}. + * True, if the wrapped value is a subtype of {@link Number}. * - * @return true, if {@code Number} value + * @return true, if {@link Number} value */ public boolean isNumber() { return !isNull() && Number.class.isAssignableFrom(getType()); @@ -353,36 +372,45 @@ public boolean isNumber() { // Getter //---------------------------------------------------------------------------- + /** + * Returns the value as the specified type. If it is already of the same type as requested, we + * just return the value. If not, an {@link UnsupportedOperationException} is thrown. + * + * @param clazz the requested value type + * @param PropertyValue supported type + * @return value + * @throws UnsupportedOperationException when trying to get the wrong type + */ + @SuppressWarnings("unchecked") + public T get(Class clazz) throws UnsupportedOperationException { + PropertyValueStrategy strategy = PropertyValueStrategyFactory.get(clazz); + if (strategy.is(value)) { + return (T) value; + } + throw new UnsupportedOperationException("Value '" + value + "' of type " + + value.getClass().getSimpleName() + ", cannot be accessed as " + clazz.getSimpleName()); + } + /** * Returns the wrapped value as object. * * @return value or {@code null} if the value is empty */ public Object getObject() { - return isBoolean() ? getBoolean() : - isShort() ? getShort() : - isInt() ? getInt() : - isLong() ? getLong() : - isFloat() ? getFloat() : - isDouble() ? getDouble() : - isString() ? getString() : - isBigDecimal() ? getBigDecimal() : - isGradoopId() ? getGradoopId() : - isMap() ? getMap() : - isList() ? getList() : - isDate() ? getDate() : - isTime() ? getTime() : - isDateTime() ? getDateTime() : - isSet() ? getSet() : - null; + Object obj = null; + if (value != null) { + obj = get(value.getClass()); + } + return obj; } + /** * Returns the wrapped value as {@code boolean}. * * @return {@code boolean} value */ public boolean getBoolean() { - return rawBytes[1] == -1; + return get(Boolean.class); } /** @@ -391,86 +419,71 @@ public boolean getBoolean() { * @return {@code short} value */ public short getShort() { - return Bytes.toShort(rawBytes, OFFSET); + return get(Short.class); } + /** * Returns the wrapped value as {@code int}. * * @return {@code int} value */ public int getInt() { - return Bytes.toInt(rawBytes, OFFSET); + return get(Integer.class); } + /** * Returns the wrapped value as {@code long}. * * @return {@code long} value */ public long getLong() { - return Bytes.toLong(rawBytes, OFFSET); + return get(Long.class); } + /** * Returns the wrapped value as {@code float}. * * @return {@code float} value */ public float getFloat() { - return Bytes.toFloat(rawBytes, OFFSET); + return get(Float.class); } + /** * Returns the wrapped value as {@code double}. * * @return {@code double} value */ public double getDouble() { - return Bytes.toDouble(rawBytes, OFFSET); + return get(Double.class); } + /** - * Returns the wrapped value as {@code String}. + * Returns the wrapped value as {@link String}. * - * @return {@code String} value + * @return {@link String} value */ public String getString() { - return Bytes.toString(rawBytes, OFFSET, rawBytes.length - OFFSET); + return get(String.class); } + /** - * Returns the wrapped value as {@code BigDecimal}. + * Returns the wrapped value as {@link BigDecimal}. * - * @return {@code BigDecimal} value + * @return {@link BigDecimal} value * @see BigDecimal */ public BigDecimal getBigDecimal() { - BigDecimal decimal; - - if (isBigDecimal()) { - decimal = Bytes.toBigDecimal(rawBytes, OFFSET, rawBytes.length - OFFSET); - } else if (isFloat()) { - decimal = BigDecimal.valueOf(getFloat()); - } else if (isDouble()) { - decimal = BigDecimal.valueOf(getDouble()); - } else if (isShort()) { - decimal = BigDecimal.valueOf(getShort()); - } else if (isInt()) { - decimal = BigDecimal.valueOf(getInt()); - } else if (isLong()) { - decimal = BigDecimal.valueOf(getLong()); - } else if (isString()) { - decimal = new BigDecimal(getString()); - } else { - throw new ClassCastException( - "Cannot covert " + this.getType().getSimpleName() + - " to " + BigDecimal.class.getSimpleName()); - } - return decimal; + return get(BigDecimal.class); } + /** - * Returns the wrapped value as {@code GradoopId}. + * Returns the wrapped value as {@link GradoopId}. * - * @return {@code GradoopId} value + * @return {@link GradoopId} value */ public GradoopId getGradoopId() { - return GradoopId.fromByteArray( - Arrays.copyOfRange(rawBytes, OFFSET, GradoopId.ID_SIZE + OFFSET)); + return get(GradoopId.class); } /** @@ -479,33 +492,7 @@ public GradoopId getGradoopId() { * @return {@code Map} value */ public Map getMap() { - PropertyValue key; - PropertyValue value; - - Map map = new HashMap<>(); - - ByteArrayInputStream byteStream = new ByteArrayInputStream(rawBytes); - DataInputStream inputStream = new DataInputStream(byteStream); - DataInputView inputView = new DataInputViewStreamWrapper(inputStream); - - try { - if (inputStream.skipBytes(OFFSET) != OFFSET) { - throw new RuntimeException("Malformed entry in PropertyValue List"); - } - while (inputStream.available() > 0) { - key = new PropertyValue(); - key.read(inputView); - - value = new PropertyValue(); - value.read(inputView); - - map.put(key, value); - } - } catch (IOException e) { - throw new RuntimeException("Error reading PropertyValue"); - } - - return map; + return get(Map.class); } /** @@ -514,86 +501,43 @@ public Map getMap() { * @return {@code List} value */ public List getList() { - PropertyValue entry; - - List list = new ArrayList<>(); - - ByteArrayInputStream byteStream = new ByteArrayInputStream(rawBytes); - DataInputStream inputStream = new DataInputStream(byteStream); - DataInputView inputView = new DataInputViewStreamWrapper(inputStream); - - try { - if (inputStream.skipBytes(OFFSET) != OFFSET) { - throw new RuntimeException("Malformed entry in PropertyValue List"); - } - while (inputStream.available() > 0) { - entry = new PropertyValue(); - entry.read(inputView); - - list.add(entry); - } - } catch (IOException e) { - throw new RuntimeException("Error reading PropertyValue"); - } - - return list; + return get(List.class); } + /** - * Returns the wrapped List as {@code LocalDate}. + * Returns the wrapped List as {@link LocalDate}. * - * @return {@code LocalDate} value + * @return {@link LocalDate} value */ public LocalDate getDate() { - return DateTimeSerializer.deserializeDate( - Arrays.copyOfRange(rawBytes, OFFSET, DateTimeSerializer.SIZEOF_DATE + OFFSET)); + return get(LocalDate.class); } + /** - * Returns the wrapped List as {@code LocalTime}. + * Returns the wrapped List as {@link LocalTime}. * - * @return {@code LocalTime} value + * @return {@link LocalTime} value */ public LocalTime getTime() { - return DateTimeSerializer.deserializeTime( - Arrays.copyOfRange(rawBytes, OFFSET, DateTimeSerializer.SIZEOF_TIME + OFFSET)); + return get(LocalTime.class); } + /** - * Returns the wrapped List as {@code LocalDateTime}. + * Returns the wrapped List as {@link LocalDateTime}. * - * @return {@code LocalDateTime} value + * @return {@link LocalDateTime} value */ public LocalDateTime getDateTime() { - return DateTimeSerializer.deserializeDateTime( - Arrays.copyOfRange(rawBytes, OFFSET, DateTimeSerializer.SIZEOF_DATETIME + OFFSET)); + return get(LocalDateTime.class); } + /** * Returns the wrapped Set as {@code Set}. * * @return {@code Set} value */ public Set getSet() { - PropertyValue entry; - - Set set = new HashSet<>(); - - ByteArrayInputStream byteStream = new ByteArrayInputStream(rawBytes); - DataInputStream inputStream = new DataInputStream(byteStream); - DataInputView inputView = new DataInputViewStreamWrapper(inputStream); - - try { - if (inputStream.skipBytes(OFFSET) != OFFSET) { - throw new RuntimeException("Malformed entry in PropertyValue Set"); - } - while (inputStream.available() > 0) { - entry = new PropertyValue(); - entry.read(inputView); - - set.add(entry); - } - } catch (IOException e) { - throw new RuntimeException("Error reading PropertyValue"); - } - - return set; + return get(Set.class); } //---------------------------------------------------------------------------- @@ -607,51 +551,19 @@ public Set getSet() { * @throws UnsupportedTypeException if the type of the Object is not supported */ public void setObject(Object value) { - if (value == null) { - rawBytes = new byte[] {TYPE_NULL}; - } else if (value instanceof Boolean) { - setBoolean((Boolean) value); - } else if (value instanceof Short) { - setShort((Short) value); - } else if (value instanceof Integer) { - setInt((Integer) value); - } else if (value instanceof Long) { - setLong((Long) value); - } else if (value instanceof Float) { - setFloat((Float) value); - } else if (value instanceof Double) { - setDouble((Double) value); - } else if (value instanceof String) { - setString((String) value); - } else if (value instanceof BigDecimal) { - setBigDecimal((BigDecimal) value); - } else if (value instanceof GradoopId) { - setGradoopId((GradoopId) value); - } else if (value instanceof Map) { - setMap((Map) value); - } else if (value instanceof List) { - setList((List) value); - } else if (value instanceof LocalDate) { - setDate((LocalDate) value); - } else if (value instanceof LocalTime) { - setTime((LocalTime) value); - } else if (value instanceof LocalDateTime) { - setDateTime((LocalDateTime) value); - } else if (value instanceof Set) { - setSet((Set) value); - } else { + if (value != null && !PropertyValueStrategyFactory.get(value.getClass()).is(value)) { throw new UnsupportedTypeException(value.getClass()); } + this.value = value; } + /** * Sets the wrapped value as {@code boolean} value. * * @param booleanValue value */ public void setBoolean(boolean booleanValue) { - rawBytes = new byte[OFFSET + Bytes.SIZEOF_BOOLEAN]; - rawBytes[0] = TYPE_BOOLEAN; - Bytes.putByte(rawBytes, OFFSET, (byte) (booleanValue ? -1 : 0)); + setObject(booleanValue); } /** @@ -660,169 +572,115 @@ public void setBoolean(boolean booleanValue) { * @param shortValue value */ public void setShort(short shortValue) { - rawBytes = new byte[OFFSET + Bytes.SIZEOF_SHORT]; - rawBytes[0] = TYPE_SHORT; - Bytes.putShort(rawBytes, OFFSET, shortValue); + setObject(shortValue); } + /** * Sets the wrapped value as {@code int} value. * * @param intValue intValue */ public void setInt(int intValue) { - rawBytes = new byte[OFFSET + Bytes.SIZEOF_INT]; - rawBytes[0] = TYPE_INTEGER; - Bytes.putInt(rawBytes, OFFSET, intValue); + setObject(intValue); } + /** * Sets the wrapped value as {@code long} value. * * @param longValue value */ public void setLong(long longValue) { - rawBytes = new byte[OFFSET + Bytes.SIZEOF_LONG]; - rawBytes[0] = TYPE_LONG; - Bytes.putLong(rawBytes, OFFSET, longValue); + setObject(longValue); } + /** * Sets the wrapped value as {@code float} value. * * @param floatValue value */ public void setFloat(float floatValue) { - rawBytes = new byte[OFFSET + Bytes.SIZEOF_FLOAT]; - rawBytes[0] = TYPE_FLOAT; - Bytes.putFloat(rawBytes, OFFSET, floatValue); + setObject(floatValue); } + /** * Sets the wrapped value as {@code double} value. * * @param doubleValue value */ public void setDouble(double doubleValue) { - rawBytes = new byte[OFFSET + Bytes.SIZEOF_DOUBLE]; - rawBytes[0] = TYPE_DOUBLE; - Bytes.putDouble(rawBytes, OFFSET, doubleValue); + setObject(doubleValue); } + /** - * Sets the wrapped value as {@code String} value. + * Sets the wrapped value as {@link String} value. * * @param stringValue value */ public void setString(String stringValue) { - byte[] valueBytes = Bytes.toBytes(stringValue); - rawBytes = new byte[OFFSET + valueBytes.length]; - rawBytes[0] = TYPE_STRING; - Bytes.putBytes(rawBytes, OFFSET, valueBytes, 0, valueBytes.length); + setObject(stringValue); } + /** - * Sets the wrapped value as {@code BigDecimal} value. + * Sets the wrapped value as {@link BigDecimal} value. * * @param bigDecimalValue value */ public void setBigDecimal(BigDecimal bigDecimalValue) { - byte[] valueBytes = Bytes.toBytes(bigDecimalValue); - rawBytes = new byte[OFFSET + valueBytes.length]; - rawBytes[0] = TYPE_BIG_DECIMAL; - Bytes.putBytes(rawBytes, OFFSET, valueBytes, 0, valueBytes.length); + setObject(bigDecimalValue); } + /** - * Sets the wrapped value as {@code GradoopId} value. + * Sets the wrapped value as {@link GradoopId} value. * * @param gradoopIdValue value */ public void setGradoopId(GradoopId gradoopIdValue) { - byte[] valueBytes = gradoopIdValue.toByteArray(); - rawBytes = new byte[OFFSET + GradoopId.ID_SIZE]; - rawBytes[0] = TYPE_GRADOOP_ID; - Bytes.putBytes(rawBytes, OFFSET, valueBytes, 0, valueBytes.length); + setObject(gradoopIdValue); } /** - * Sets the wrapped value as {@code Map} value. + * Sets the wrapped value as {@link Map} value. * * @param map value */ public void setMap(Map map) { - int size = - map.keySet().stream().mapToInt(PropertyValue::byteSize).sum() + - map.values().stream().mapToInt(PropertyValue::byteSize).sum() + - OFFSET; - - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(size); - DataOutputStream outputStream = new DataOutputStream(byteStream); - DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream); - - try { - outputStream.write(TYPE_MAP); - for (Map.Entry entry : map.entrySet()) { - entry.getKey().write(outputView); - entry.getValue().write(outputView); - } - } catch (IOException e) { - throw new RuntimeException("Error writing PropertyValue"); - } - - this.rawBytes = byteStream.toByteArray(); + setObject(map); } /** - * Sets the wrapped value as {@code List} value. + * Sets the wrapped value as {@link List} value. * * @param list value */ public void setList(List list) { - int size = list.stream().mapToInt(PropertyValue::byteSize).sum() + - OFFSET; - - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(size); - DataOutputStream outputStream = new DataOutputStream(byteStream); - DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream); - - try { - outputStream.write(TYPE_LIST); - for (PropertyValue entry : list) { - entry.write(outputView); - } - } catch (IOException e) { - throw new RuntimeException("Error writing PropertyValue"); - } - - this.rawBytes = byteStream.toByteArray(); + setObject(list); } /** - * Sets the wrapped value as {@code LocalDate} value. + * Sets the wrapped value as {@link LocalDate} value. * * @param date value */ public void setDate(LocalDate date) { - byte[] valueBytes = DateTimeSerializer.serializeDate(date); - rawBytes = new byte[OFFSET + DateTimeSerializer.SIZEOF_DATE]; - rawBytes[0] = TYPE_DATE; - Bytes.putBytes(rawBytes, OFFSET, valueBytes, 0, valueBytes.length); + setObject(date); } + /** - * Sets the wrapped value as {@code LocalTime} value. + * Sets the wrapped value as {@link LocalTime} value. * * @param time value */ public void setTime(LocalTime time) { - byte[] valueBytes = DateTimeSerializer.serializeTime(time); - rawBytes = new byte[OFFSET + DateTimeSerializer.SIZEOF_TIME]; - rawBytes[0] = TYPE_TIME; - Bytes.putBytes(rawBytes, OFFSET, valueBytes, 0, valueBytes.length); + setObject(time); } + /** * Sets the wrapped value as {@code LocalDateTime} value. * * @param dateTime value */ public void setDateTime(LocalDateTime dateTime) { - byte[] valueBytes = DateTimeSerializer.serializeDateTime(dateTime); - rawBytes = new byte[OFFSET + DateTimeSerializer.SIZEOF_DATETIME]; - rawBytes[0] = TYPE_DATETIME; - Bytes.putBytes(rawBytes, OFFSET, valueBytes, 0, valueBytes.length); + setObject(dateTime); } /** @@ -831,22 +689,7 @@ public void setDateTime(LocalDateTime dateTime) { * @param set value */ public void setSet(Set set) { - int size = set.stream().mapToInt(PropertyValue::byteSize).sum() + OFFSET; - - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(size); - DataOutputStream outputStream = new DataOutputStream(byteStream); - DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream); - - try { - outputStream.write(TYPE_SET); - for (PropertyValue entry : set) { - entry.write(outputView); - } - } catch (IOException e) { - throw new RuntimeException("Error writing PropertyValue"); - } - - this.rawBytes = byteStream.toByteArray(); + setObject(set); } //---------------------------------------------------------------------------- @@ -854,121 +697,58 @@ public void setSet(Set set) { //---------------------------------------------------------------------------- /** - * Get the data type as class object according to the first position of the rawBytes[] array + * Get the data type as class object according to the first position of the rawBytes[] array. * * @return Class object */ public Class getType() { - return TYPE_MAPPING.get(rawBytes[0]); - } + Class clazz = null; + if (value != null) { + clazz = PropertyValueStrategyFactory.get(value.getClass()).getType(); + } - /** - * Creates a type mapping HashMap to assign a byte value to its represented Class - * - * @return a Map with byte to class assignments - */ - private static Map getTypeMap() { - Map map = new HashMap<>(); - map.put(TYPE_BOOLEAN, Boolean.class); - map.put(TYPE_SHORT, Short.class); - map.put(TYPE_INTEGER, Integer.class); - map.put(TYPE_LONG, Long.class); - map.put(TYPE_FLOAT, Float.class); - map.put(TYPE_DOUBLE, Double.class); - map.put(TYPE_STRING, String.class); - map.put(TYPE_BIG_DECIMAL, BigDecimal.class); - map.put(TYPE_GRADOOP_ID, GradoopId.class); - map.put(TYPE_MAP, Map.class); - map.put(TYPE_LIST, List.class); - map.put(TYPE_DATE, LocalDate.class); - map.put(TYPE_TIME, LocalTime.class); - map.put(TYPE_DATETIME, LocalDateTime.class); - map.put(TYPE_SET, Set.class); - return Collections.unmodifiableMap(map); + return clazz; } public int getByteSize() { - return rawBytes.length; + return getRawBytes().length; } - @SuppressWarnings("EI_EXPOSE_REP") public byte[] getRawBytes() { - return this.rawBytes; + return PropertyValueStrategyFactory.getRawBytes(value); } /** - * Set internal byte representation + * Set internal byte representation. + * * @param bytes array */ - @SuppressWarnings("EI_EXPOSE_REP") public void setBytes(byte[] bytes) { - this.rawBytes = bytes; - } - - /** - * Create a {@link PropertyValue} that wraps a byte array - * @param rawBytes array to wrap - * @return new instance of {@link PropertyValue} - */ - public static PropertyValue fromRawBytes(byte[] rawBytes) { - return new PropertyValue(rawBytes); + value = PropertyValueStrategyFactory.fromRawBytes(bytes); } @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof PropertyValue)) { - return false; - } - PropertyValue that = (PropertyValue) o; - return Arrays.equals(rawBytes, that.rawBytes); + public boolean equals(Object object) { + return object instanceof PropertyValue && Objects.equals(value, ((PropertyValue) object).value); } @Override public int hashCode() { - return Arrays.hashCode(rawBytes); + return Arrays.hashCode(PropertyValueStrategyFactory.getRawBytes(value)); } @Override - public int compareTo(PropertyValue o) { - int result; - - if (this.isNull() && o.isNull()) { - result = 0; - } else if (this.isNumber() && o.isNumber()) { - result = PropertyValueUtils.Numeric.compare(this, o); - } else if (this.isBoolean() && o.isBoolean()) { - result = Boolean.compare(this.getBoolean(), o.getBoolean()); - } else if (this.isString() && o.isString()) { - result = this.getString().compareTo(o.getString()); - } else if (this.isGradoopId() && o.isGradoopId()) { - result = this.getGradoopId().compareTo(o.getGradoopId()); - } else if (this.isDate() && o.isDate()) { - result = this.getDate().compareTo(o.getDate()); - } else if (this.isTime() && o.isTime()) { - result = this.getTime().compareTo(o.getTime()); - } else if (this.isDateTime() && o.isDateTime()) { - result = this.getDateTime().compareTo(o.getDateTime()); - } else if (this.isMap() || o.isMap() || - this.isList() || o.isList() || - this.isSet() || o.isSet()) { - throw new UnsupportedOperationException(String.format( - "Method compareTo() is not supported for %s, %s", this.getClass(), o.getClass())); - } else { - throw new IllegalArgumentException(String.format( - "Incompatible types: %s, %s", this.getClass(), o.getClass())); - } - - return result; + public int compareTo(PropertyValue other) { + return PropertyValueStrategyFactory.compare(value, other.value); } /** - * Returns the byte size of the properties internal representation + * Returns the byte size of the properties internal representation. + * * @return byte size */ public int byteSize() { + byte[] rawBytes = PropertyValueStrategyFactory.getRawBytes(value); return rawBytes.length; } @@ -996,76 +776,26 @@ public int byteSize() { * byte 2 - end : value bytes * * @param outputView data output to write data to - * @throws IOException + * @throws IOException if write to output view fails. */ @Override public void write(DataOutputView outputView) throws IOException { - // null? - // Write type. - byte type = rawBytes[0]; - if (rawBytes.length > LARGE_PROPERTY_THRESHOLD) { - type |= FLAG_LARGE; - } - outputView.writeByte(type); - // Write length for types with a variable length. - if (isString() || isBigDecimal() || isMap() || isList() || isSet()) { - // Write length as an int if the "large" flag is set. - if ((type & FLAG_LARGE) == FLAG_LARGE) { - outputView.writeInt(rawBytes.length - OFFSET); - } else { - outputView.writeShort(rawBytes.length - OFFSET); - } - } - // write data - outputView.write(rawBytes, OFFSET, rawBytes.length - OFFSET); + PropertyValueStrategyFactory.get(value).write(value, outputView); } @Override public void read(DataInputView inputView) throws IOException { - int length = 0; // type byte typeByte = inputView.readByte(); // Apply bitmask to get the actual type. - byte type = (byte) (~FLAG_LARGE & typeByte); - // dynamic type? - if (type == TYPE_STRING || type == TYPE_BIG_DECIMAL || type == TYPE_MAP || - type == TYPE_LIST || type == TYPE_SET) { - // read length - if ((typeByte & FLAG_LARGE) == FLAG_LARGE) { - length = inputView.readInt(); - } else { - length = inputView.readShort(); - } - } else if (type == TYPE_NULL) { - length = 0; - } else if (type == TYPE_BOOLEAN) { - length = Bytes.SIZEOF_BOOLEAN; - } else if (type == TYPE_SHORT) { - length = Bytes.SIZEOF_SHORT; - } else if (type == TYPE_INTEGER) { - length = Bytes.SIZEOF_INT; - } else if (type == TYPE_LONG) { - length = Bytes.SIZEOF_LONG; - } else if (type == TYPE_FLOAT) { - length = Bytes.SIZEOF_FLOAT; - } else if (type == TYPE_DOUBLE) { - length = Bytes.SIZEOF_DOUBLE; - } else if (type == TYPE_GRADOOP_ID) { - length = GradoopId.ID_SIZE; - } else if (type == TYPE_DATE) { - length = DateTimeSerializer.SIZEOF_DATE; - } else if (type == TYPE_TIME) { - length = DateTimeSerializer.SIZEOF_TIME; - } else if (type == TYPE_DATETIME) { - length = DateTimeSerializer.SIZEOF_DATETIME; - } - // init new array - rawBytes = new byte[OFFSET + length]; - // read type info - rawBytes[0] = type; - // read data - for (int i = OFFSET; i < rawBytes.length; i++) { - rawBytes[i] = inputView.readByte(); + byte type = (byte) (~PropertyValue.FLAG_LARGE & typeByte); + + PropertyValueStrategy strategy = PropertyValueStrategyFactory.get(type); + + if (strategy == null) { + throw new UnsupportedTypeException("No strategy for type byte from input view found"); + } else { + value = strategy.read(inputView, typeByte); } } diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/PropertyValueList.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/PropertyValueList.java index a592881dfe44..154b24f7f0d3 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/PropertyValueList.java +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/PropertyValueList.java @@ -77,8 +77,8 @@ public static PropertyValueList createEmptyList() { * Creates a Property value list from a collection of property values. * * @param propertyValues property values - * * @return property value list containing the given properties + * @throws IOException on failure */ public static PropertyValueList fromPropertyValues( Collection propertyValues) throws IOException { diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/AbstractFixSizedPropertyValueStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/AbstractFixSizedPropertyValueStrategy.java new file mode 100644 index 000000000000..9ea12e321dc6 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/AbstractFixSizedPropertyValueStrategy.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataOutputView; +import org.gradoop.common.model.api.strategies.PropertyValueStrategy; + +import java.io.IOException; + +/** + * Abstract class that provides generic methods for {@code PropertyValueStrategy} classes that + * handle data types with a fixed size. + * + * @param Type with a fixed length. + */ +public abstract class AbstractFixSizedPropertyValueStrategy implements PropertyValueStrategy { + + @Override + public void write(T value, DataOutputView outputView) throws IOException { + outputView.write(getRawBytes(value)); + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/AbstractVariableSizedPropertyValueStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/AbstractVariableSizedPropertyValueStrategy.java new file mode 100644 index 000000000000..fd4416b071f1 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/AbstractVariableSizedPropertyValueStrategy.java @@ -0,0 +1,94 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputView; +import org.gradoop.common.model.api.strategies.PropertyValueStrategy; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; + +/** + * Abstract class that provides generic methods for {@code PropertyValueStrategy} classes that + * handle data types with a variable size. + * + * @param Type with a variable length. + */ +public abstract class AbstractVariableSizedPropertyValueStrategy implements PropertyValueStrategy { + + @Override + public void write(T value, DataOutputView outputView) throws IOException { + byte[] rawBytes = getRawBytes(value); + byte type = rawBytes[0]; + + if (rawBytes.length > PropertyValue.LARGE_PROPERTY_THRESHOLD) { + type |= PropertyValue.FLAG_LARGE; + } + outputView.writeByte(type); + + // Write length as an int if the "large" flag is set. + if ((type & PropertyValue.FLAG_LARGE) == PropertyValue.FLAG_LARGE) { + outputView.writeInt(rawBytes.length - PropertyValue.OFFSET); + } else { + outputView.writeShort(rawBytes.length - PropertyValue.OFFSET); + } + + outputView.write(rawBytes, PropertyValue.OFFSET, rawBytes.length - PropertyValue.OFFSET); + } + + /** + * Reads data of variable size from a data input view. The size of the data is determined by the + * type byte and {@see org.gradoop.common.model.impl.properties.PropertyValue#FLAG_LARGE}. + * + * @param inputView Data input view to read from + * @param typeByte Byte indicating the type of the serialized data + * @return The serialized data in the data input view + * @throws IOException when reading a byte goes wrong + */ + byte[] readVariableSizedData(DataInputView inputView, byte typeByte) throws IOException { + int length; + // read length + if ((typeByte & PropertyValue.FLAG_LARGE) == PropertyValue.FLAG_LARGE) { + length = inputView.readInt(); + } else { + length = inputView.readShort(); + } + // init new array + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return rawBytes; + } + + /** + * Creates an instance of {@link DataInputViewStreamWrapper} from a byte array. + * + * @param bytes input byte array + * @return A DataInputViewStreamWrapper + */ + DataInputViewStreamWrapper createInputView(byte[] bytes) { + ByteArrayInputStream byteStream = new ByteArrayInputStream(bytes); + DataInputStream inputStream = new DataInputStream(byteStream); + return new DataInputViewStreamWrapper(inputStream); + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/BigDecimalStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/BigDecimalStrategy.java new file mode 100644 index 000000000000..c9a8ba76e52d --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/BigDecimalStrategy.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.io.IOException; +import java.math.BigDecimal; + +/** + * Strategy class for handling {@link PropertyValue} operations with a value of the type + * {@link BigDecimal}. + */ +public class BigDecimalStrategy extends AbstractVariableSizedPropertyValueStrategy { + + @Override + public BigDecimal read(DataInputView inputView, byte typeByte) throws IOException { + byte[] rawBytes = readVariableSizedData(inputView, typeByte); + return Bytes.toBigDecimal(rawBytes); + } + + @Override + public int compare(BigDecimal value, Object other) { + if (other instanceof Number) { + Number num = (Number) other; + return PropertyValueStrategyUtils.compareNumerical(value, num); + } + throw new IllegalArgumentException(String.format( + "Incompatible types: %s, %s", value.getClass(), other.getClass())); + } + + @Override + public boolean is(Object value) { + return value instanceof BigDecimal; + } + + @Override + public Class getType() { + return BigDecimal.class; + } + + @Override + public BigDecimal get(byte[] bytes) { + return Bytes.toBigDecimal(bytes, PropertyValue.OFFSET, bytes.length - PropertyValue.OFFSET); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_BIG_DECIMAL; + } + + @Override + public byte[] getRawBytes(BigDecimal value) { + byte[] valueBytes = Bytes.toBytes(value); + byte[] rawBytes = new byte[PropertyValue.OFFSET + valueBytes.length]; + rawBytes[0] = getRawType(); + Bytes.putBytes(rawBytes, PropertyValue.OFFSET, valueBytes, 0, valueBytes.length); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/BooleanStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/BooleanStrategy.java new file mode 100644 index 000000000000..cd3cd964a3d1 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/BooleanStrategy.java @@ -0,0 +1,77 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.io.IOException; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code Boolean}. + */ +public class BooleanStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public Boolean read(DataInputView inputView, byte typeByte) throws IOException { + return inputView.readByte() == -1; + } + + @Override + public int compare(Boolean value, Object other) { + if (other.getClass() == Boolean.class) { + return Boolean.compare(value, (boolean) other); + } + throw new IllegalArgumentException(String.format( + "Incompatible types: %s, %s", value.getClass(), other.getClass())); + } + + @Override + public boolean is(Object value) { + return value instanceof Boolean; + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public Boolean get(byte[] bytes) { + return bytes[1] == -1; + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_BOOLEAN; + } + + /** + * {@inheritDoc} + * + * Warning: Please note that if {@code null} is passed as an argument, it is going to be evaluated + * as if it was false. + */ + @Override + public byte[] getRawBytes(Boolean value) { + byte[] rawBytes = new byte[PropertyValue.OFFSET + Bytes.SIZEOF_BOOLEAN]; + rawBytes[0] = getRawType(); + Bytes.putByte(rawBytes, PropertyValue.OFFSET, (byte) (value ? -1 : 0)); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DateStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DateStrategy.java new file mode 100644 index 000000000000..7b1226a3a310 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DateStrategy.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.DateTimeSerializer; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; +import java.time.LocalDate; +import java.util.Arrays; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code LocalDate}. + */ +public class DateStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public LocalDate read(DataInputView inputView, byte typeByte) throws IOException { + int length = DateTimeSerializer.SIZEOF_DATE; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return DateTimeSerializer.deserializeDate(rawBytes); + } + + @Override + public int compare(LocalDate value, Object other) { + if (other instanceof LocalDate) { + return value.compareTo((LocalDate) other); + } + throw new IllegalArgumentException(String.format( + "Incompatible types: %s, %s", value.getClass(), other.getClass())); + } + + @Override + public boolean is(Object value) { + return value instanceof LocalDate; + } + + @Override + public Class getType() { + return LocalDate.class; + } + + @Override + public LocalDate get(byte[] bytes) { + return DateTimeSerializer.deserializeDate( + Arrays.copyOfRange( + bytes, PropertyValue.OFFSET, DateTimeSerializer.SIZEOF_DATE + PropertyValue.OFFSET + )); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_DATE; + } + + @Override + public byte[] getRawBytes(LocalDate value) { + byte[] valueBytes = DateTimeSerializer.serializeDate(value); + byte[] rawBytes = new byte[PropertyValue.OFFSET + DateTimeSerializer.SIZEOF_DATE]; + rawBytes[0] = getRawType(); + Bytes.putBytes(rawBytes, PropertyValue.OFFSET, valueBytes, 0, valueBytes.length); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DateTimeStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DateTimeStrategy.java new file mode 100644 index 000000000000..5a4d651c05a4 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DateTimeStrategy.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.DateTimeSerializer; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.Arrays; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code LocalDateTime}. + */ +public class DateTimeStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public LocalDateTime read(DataInputView inputView, byte typeByte) throws IOException { + int length = DateTimeSerializer.SIZEOF_DATETIME; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return DateTimeSerializer.deserializeDateTime(rawBytes); + } + + @Override + public int compare(LocalDateTime value, Object other) { + if (other instanceof LocalDateTime) { + return value.compareTo((LocalDateTime) other); + } + throw new IllegalArgumentException(String.format( + "Incompatible types: %s, %s", value.getClass(), other.getClass())); + } + + @Override + public boolean is(Object value) { + return value instanceof LocalDateTime; + } + + @Override + public Class getType() { + return LocalDateTime.class; + } + + @Override + public LocalDateTime get(byte[] bytes) { + return DateTimeSerializer.deserializeDateTime( + Arrays.copyOfRange( + bytes, PropertyValue.OFFSET, DateTimeSerializer.SIZEOF_DATETIME + PropertyValue.OFFSET + )); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_DATETIME; + } + + @Override + public byte[] getRawBytes(LocalDateTime value) { + byte[] valueBytes = DateTimeSerializer.serializeDateTime(value); + byte[] rawBytes = new byte[PropertyValue.OFFSET + DateTimeSerializer.SIZEOF_DATETIME]; + rawBytes[0] = getRawType(); + Bytes.putBytes(rawBytes, PropertyValue.OFFSET, valueBytes, 0, valueBytes.length); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DoubleStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DoubleStrategy.java new file mode 100644 index 000000000000..ddfee67abe80 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/DoubleStrategy.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code Double}. + */ +public class DoubleStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public Double read(DataInputView inputView, byte typeByte) throws IOException { + int length = Bytes.SIZEOF_DOUBLE; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return Bytes.toDouble(rawBytes); + } + + @Override + public int compare(Double value, Object other) { + return PropertyValueStrategyUtils.compareNumerical(value, other); + } + + @Override + public boolean is(Object value) { + return value instanceof Double; + } + + @Override + public Class getType() { + return Double.class; + } + + @Override + public Double get(byte[] bytes) { + return Bytes.toDouble(bytes, PropertyValue.OFFSET); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_DOUBLE; + } + + @Override + public byte[] getRawBytes(Double value) { + byte[] rawBytes = new byte[PropertyValue.OFFSET + Bytes.SIZEOF_DOUBLE]; + rawBytes[0] = getRawType(); + Bytes.putDouble(rawBytes, PropertyValue.OFFSET, value); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/FloatStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/FloatStrategy.java new file mode 100644 index 000000000000..f75e8fc3cb98 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/FloatStrategy.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code Float}. + */ +public class FloatStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public Float read(DataInputView inputView, byte typeByte) throws IOException { + int length = Bytes.SIZEOF_FLOAT; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return Bytes.toFloat(rawBytes); + } + + @Override + public int compare(Float value, Object other) { + return PropertyValueStrategyUtils.compareNumerical(value, other); + } + + @Override + public boolean is(Object value) { + return value instanceof Float; + } + + @Override + public Class getType() { + return Float.class; + } + + @Override + public Float get(byte[] bytes) { + return Bytes.toFloat(bytes, PropertyValue.OFFSET); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_FLOAT; + } + + @Override + public byte[] getRawBytes(Float value) { + byte[] rawBytes = new byte[PropertyValue.OFFSET + Bytes.SIZEOF_FLOAT]; + rawBytes[0] = getRawType(); + Bytes.putFloat(rawBytes, PropertyValue.OFFSET, value); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/GradoopIdStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/GradoopIdStrategy.java new file mode 100644 index 000000000000..634203ed59e7 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/GradoopIdStrategy.java @@ -0,0 +1,83 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; +import java.util.Arrays; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code GradoopIdStrategy}. + */ +public class GradoopIdStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public GradoopId read(DataInputView inputView, byte typeByte) throws IOException { + int length = GradoopId.ID_SIZE; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return GradoopId.fromByteArray(rawBytes); + } + + @Override + public int compare(GradoopId value, Object other) { + if (other instanceof GradoopId) { + return value.compareTo((GradoopId) other); + } + throw new IllegalArgumentException(String.format( + "Incompatible types: %s, %s", value.getClass(), other.getClass())); + } + + @Override + public boolean is(Object value) { + return value instanceof GradoopId; + } + + @Override + public Class getType() { + return GradoopId.class; + } + + @Override + public GradoopId get(byte[] bytes) { + return GradoopId.fromByteArray( + Arrays.copyOfRange( + bytes, PropertyValue.OFFSET, GradoopId.ID_SIZE + PropertyValue.OFFSET + )); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_GRADOOP_ID; + } + + @Override + public byte[] getRawBytes(GradoopId value) { + byte[] valueBytes = value.toByteArray(); + byte[] rawBytes = new byte[PropertyValue.OFFSET + GradoopId.ID_SIZE]; + rawBytes[0] = getRawType(); + Bytes.putBytes(rawBytes, PropertyValue.OFFSET, valueBytes, 0, valueBytes.length); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/IntegerStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/IntegerStrategy.java new file mode 100644 index 000000000000..746e60712658 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/IntegerStrategy.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code Integer}. + */ +public class IntegerStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public Integer read(DataInputView inputView, byte typeByte) throws IOException { + int length = Bytes.SIZEOF_INT; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return Bytes.toInt(rawBytes); + } + + @Override + public int compare(Integer value, Object other) { + return PropertyValueStrategyUtils.compareNumerical(value, other); + } + + @Override + public boolean is(Object value) { + return value instanceof Integer; + } + + @Override + public Class getType() { + return Integer.class; + } + + @Override + public Integer get(byte[] bytes) { + return Bytes.toInt(bytes, PropertyValue.OFFSET); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_INTEGER; + } + + @Override + public byte[] getRawBytes(Integer value) { + byte[] rawBytes = new byte[PropertyValue.OFFSET + Bytes.SIZEOF_INT]; + rawBytes[0] = getRawType(); + Bytes.putInt(rawBytes, PropertyValue.OFFSET, value); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/ListStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/ListStrategy.java new file mode 100644 index 000000000000..9cff78b9594d --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/ListStrategy.java @@ -0,0 +1,141 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code List}. + */ +public class ListStrategy extends AbstractVariableSizedPropertyValueStrategy> { + + @Override + public List read(DataInputView inputView, byte typeByte) throws IOException { + byte[] rawBytes = readVariableSizedData(inputView, typeByte); + + PropertyValue item; + List list = new ArrayList<>(); + DataInputViewStreamWrapper internalInputView = createInputView(rawBytes); + + try { + while (internalInputView.available() > 0) { + item = new PropertyValue(); + item.read(internalInputView); + + list.add(item); + } + } catch (IOException e) { + throw new IOException("Error reading PropertyValue with ListStrategy.", e); + } + + return list; + } + + @Override + public int compare(List value, Object other) { + throw new UnsupportedOperationException("Method compare() is not supported for List."); + } + + @Override + public boolean is(Object value) { + if (!(value instanceof List)) { + return false; + } + for (Object item : (List) value) { + // List is not a valid property value if it contains any object + // that is not a property value itself. + if (!(item instanceof PropertyValue)) { + return false; + } + } + + return true; + } + + @Override + public Class> getType() { + return (Class) List.class; + } + + /** + * {@inheritDoc} + * @throws IOException if converting the byte array to a List fails. + */ + @Override + public List get(byte[] bytes) throws IOException { + PropertyValue item; + List list = new ArrayList<>(); + + try (ByteArrayInputStream byteStream = new ByteArrayInputStream(bytes); + DataInputStream inputStream = new DataInputStream(byteStream); + DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(inputStream)) { + + if (inputView.skipBytes(PropertyValue.OFFSET) != PropertyValue.OFFSET) { + throw new IOException("Malformed entry in PropertyValue List."); + } + while (inputView.available() > 0) { + item = new PropertyValue(); + item.read(inputView); + + list.add(item); + } + } catch (IOException e) { + throw new IOException("Error reading PropertyValue with ListStrategy.", e); + } + + return list; + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_LIST; + } + + /** + * {@inheritDoc} + * @throws IOException if converting the value to a byte array fails. + */ + @Override + public byte[] getRawBytes(List value) throws IOException { + int size = value.stream().mapToInt(PropertyValue::byteSize).sum() + + PropertyValue.OFFSET; + + try (ByteArrayOutputStream byteStream = new ByteArrayOutputStream(size); + DataOutputStream outputStream = new DataOutputStream(byteStream); + DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream)) { + + outputStream.write(getRawType()); + for (PropertyValue entry : value) { + entry.write(outputView); + } + + return byteStream.toByteArray(); + } catch (IOException e) { + throw new IOException("Error writing PropertyValue with ListStrategy.", e); + } + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/LongStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/LongStrategy.java new file mode 100644 index 000000000000..c282f7fc492e --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/LongStrategy.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code Long}. + */ +public class LongStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public Long read(DataInputView inputView, byte typeByte) throws IOException { + int length = Bytes.SIZEOF_LONG; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return Bytes.toLong(rawBytes); + } + + @Override + public int compare(Long value, Object other) { + return PropertyValueStrategyUtils.compareNumerical(value, other); + } + + @Override + public boolean is(Object value) { + return value instanceof Long; + } + + @Override + public Class getType() { + return Long.class; + } + + @Override + public Long get(byte[] bytes) { + return Bytes.toLong(bytes, PropertyValue.OFFSET); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_LONG; + } + + @Override + public byte[] getRawBytes(Long value) { + byte[] rawBytes = new byte[PropertyValue.OFFSET + Bytes.SIZEOF_LONG]; + rawBytes[0] = getRawType(); + Bytes.putLong(rawBytes, PropertyValue.OFFSET, value); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/MapStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/MapStrategy.java new file mode 100644 index 000000000000..4fcaef33c743 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/MapStrategy.java @@ -0,0 +1,150 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code Map}. + */ +public class MapStrategy + extends AbstractVariableSizedPropertyValueStrategy> { + + @Override + public Map read(DataInputView inputView, byte typeByte) + throws IOException { + byte[] rawBytes = readVariableSizedData(inputView, typeByte); + DataInputViewStreamWrapper internalInputView = createInputView(rawBytes); + return createMap(internalInputView); + } + + @Override + public int compare(Map value, Object other) { + throw new UnsupportedOperationException("Method compare() is not supported for Map."); + } + + @Override + public boolean is(Object value) { + if (!(value instanceof Map)) { + return false; + } + for (Map.Entry entry : ((Map) value).entrySet()) { + // Map is not a valid property value if it contains any object + // that is not a property value itself. + if (!(entry.getKey() instanceof PropertyValue) || + !(entry.getValue() instanceof PropertyValue)) { + return false; + } + } + + return true; + } + + @Override + public Class> getType() { + return (Class) Map.class; + } + + /** + * {@inheritDoc} + * @throws IOException if converting the byte array to a Map fails. + */ + @Override + public Map get(byte[] bytes) throws IOException { + DataInputViewStreamWrapper inputView = createInputView(bytes); + Map map; + + try { + if (inputView.skipBytes(PropertyValue.OFFSET) != PropertyValue.OFFSET) { + throw new IOException("Malformed entry in PropertyValue Map."); + } + map = createMap(inputView); + } catch (IOException e) { + throw new IOException("Error while processing DataInputViewStreamWrapper."); + } + + return map; + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_MAP; + } + + /** + * {@inheritDoc} + * @throws IOException if converting the value to a byte array fails. + */ + @Override + public byte[] getRawBytes(Map value) throws IOException { + int size = value.keySet().stream().mapToInt(PropertyValue::byteSize).sum() + + value.values().stream().mapToInt(PropertyValue::byteSize).sum() + + PropertyValue.OFFSET; + + try (ByteArrayOutputStream byteStream = new ByteArrayOutputStream(size); + DataOutputStream outputStream = new DataOutputStream(byteStream); + DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream)) { + + outputStream.write(PropertyValue.TYPE_MAP); + for (Map.Entry entry : value.entrySet()) { + entry.getKey().write(outputView); + entry.getValue().write(outputView); + } + + return byteStream.toByteArray(); + } catch (IOException e) { + throw new IOException("Error writing PropertyValue with MapStrategy.", e); + } + } + + /** + * Creates a map with data read from an {@link DataInputViewStreamWrapper}. + * + * @param inputView {@link DataInputViewStreamWrapper} containing data + * @return a map containing the deserialized data + */ + private Map createMap(DataInputViewStreamWrapper inputView) + throws IOException { + PropertyValue key; + PropertyValue value; + + Map map = new HashMap<>(); + + try { + while (inputView.available() > 0) { + key = new PropertyValue(); + key.read(inputView); + + value = new PropertyValue(); + value.read(inputView); + + map.put(key, value); + } + } catch (IOException e) { + throw new IOException("Error reading PropertyValue with MapStrategy.", e); + } + return map; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/NullStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/NullStrategy.java new file mode 100644 index 000000000000..31dd277c4816 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/NullStrategy.java @@ -0,0 +1,72 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.gradoop.common.model.api.strategies.PropertyValueStrategy; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.io.IOException; + +/** + * Strategy class for handling {@code PropertyValue} operations when the value is {@code null}. + */ +public class NullStrategy implements PropertyValueStrategy { + + @Override + public void write(Object value, DataOutputView outputView) throws IOException { + outputView.write(new byte[]{getRawType()}); + } + + @Override + public Object read(DataInputView inputView, byte typeByte) throws IOException { + return null; + } + + @Override + public int compare(Object value, Object other) { + if (value == null && other == null) { + return 0; + } + return -1; + } + + @Override + public boolean is(Object value) { + return false; + } + + @Override + public Class getType() { + return null; + } + + @Override + public Object get(byte[] bytes) { + return null; + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_NULL; + } + + @Override + public byte[] getRawBytes(Object value) { + return null; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/PropertyValueStrategyFactory.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/PropertyValueStrategyFactory.java new file mode 100644 index 000000000000..2e5269c62483 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/PropertyValueStrategyFactory.java @@ -0,0 +1,231 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.gradoop.common.exceptions.UnsupportedTypeException; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.api.strategies.PropertyValueStrategy; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Factory class responsible for instantiating strategy classes that manage every kind of access to + * the current {@link PropertyValue} value. + */ +public class PropertyValueStrategyFactory { + + /** + * {@link PropertyValueStrategyFactory} instance + */ + private static PropertyValueStrategyFactory INSTANCE = new PropertyValueStrategyFactory(); + /** + * Map which links a data type to a strategy class + */ + private final Map classStrategyMap; + /** + * Array which links a type byte (as index) to a strategy class + */ + private final PropertyValueStrategy[] byteStrategyMap; + /** + * Strategy for {@code null}-value properties + */ + private final NullStrategy nullStrategy; + + /** + * Constructs an {@link PropertyValueStrategyFactory} with type - strategy mappings as defined in + * {@code initClassStrategyMap}. + * Only one instance of this class is needed. + */ + private PropertyValueStrategyFactory() { + nullStrategy = new NullStrategy(); + classStrategyMap = initClassStrategyMap(); + byteStrategyMap = initByteStrategyMap(); + } + + /** + * Returns value which is represented by the the provided byte array. Assumes that the value is + * serialized according to the {@link PropertyValue} standard. + * + * @param bytes byte array of raw bytes. + * @return Object which is the result of the deserialization. + */ + public static Object fromRawBytes(byte[] bytes) { + PropertyValueStrategy strategy = INSTANCE.byteStrategyMap[bytes[0]]; + try { + return strategy == null ? null : strategy.get(bytes); + } catch (IOException e) { + throw new RuntimeException("Error while deserializing object.", e); + } + } + + /** + * Compares two values. + * If {@code other} is not comparable to {@code value}, the used {@link PropertyValueStrategy} + * will throw an {@code IllegalArgumentException}. + * + * @param value first value. + * @param other second value. + * @return a negative integer, zero, or a positive integer as {@code value} is less than, equal + * to, or greater than {@code other}. + */ + public static int compare(Object value, Object other) { + if (value == null) { + return INSTANCE.nullStrategy.compare(null, other); + } else { + PropertyValueStrategy strategy = get(value.getClass()); + return strategy.compare(value, other); + } + } + + /** + * Get byte array representation of the provided object. The object is serialized according to the + * {@link PropertyValue} standard. + * If the given type is not supported, an {@code UnsupportedTypeException} will be thrown. + * + * @param value to be serialized. + * @return byte array representation of the provided object. + */ + public static byte[] getRawBytes(Object value) { + if (value != null) { + try { + return get(value.getClass()).getRawBytes(value); + } catch (IOException e) { + throw new RuntimeException("Error while serializing object.", e); + } + } + return new byte[] {PropertyValue.TYPE_NULL}; + } + + /** + * Returns strategy mapping to the provided byte value. + * + * @param value representing a data type. + * @return strategy class. + * @throws UnsupportedTypeException when there is no matching strategy for a given type byte. + */ + public static PropertyValueStrategy get(byte value) throws UnsupportedTypeException { + PropertyValueStrategy strategy = INSTANCE.byteStrategyMap[value]; + if (strategy == null) { + throw new UnsupportedTypeException("No strategy for type byte " + value); + } + + return strategy; + } + + /** + * Get a strategy which corresponds the provided class. If there is no mapping for the provided + * class in the class-strategy map, or the value of the parameter is {@code null}, an instance of + * {@link NullStrategy} is returned. + * + * @param clazz some class + * @return strategy class which is able to handle the provided type. + * @throws UnsupportedTypeException when there is no matching strategy for the given class. + */ + public static PropertyValueStrategy get(Class clazz) throws UnsupportedTypeException { + if (clazz == null) { + return INSTANCE.nullStrategy; + } + + PropertyValueStrategy strategy = INSTANCE.classStrategyMap.get(clazz); + // class could be some implementation of List/Map/Set that we don't register in the class- + // strategy map, so we need to check for that. + if (strategy == null) { + if (Map.class.isAssignableFrom(clazz)) { + strategy = INSTANCE.classStrategyMap.get(Map.class); + } else if (Set.class.isAssignableFrom(clazz)) { + strategy = INSTANCE.classStrategyMap.get(Set.class); + } else if (List.class.isAssignableFrom(clazz)) { + strategy = INSTANCE.classStrategyMap.get(List.class); + } + } + if (strategy == null) { + throw new UnsupportedTypeException("No strategy for class " + clazz); + } + + return strategy; + } + + /** + * Get strategy by object. + * + * @param value some object. + * @return strategy that handles operations on the provided object type. If value is {@code null}, + * {@link NullStrategy} is returned. + * @throws UnsupportedTypeException when there is no matching strategy for a given object. + */ + public static PropertyValueStrategy get(Object value) throws UnsupportedTypeException { + if (value != null) { + PropertyValueStrategy strategy = get(value.getClass()); + if (strategy == null) { + throw new UnsupportedTypeException("No strategy for class " + value.getClass()); + } + return get(value.getClass()); + } + return INSTANCE.nullStrategy; + } + + /** + * Initializes class-strategy mapping. + * + * @return Map of supported class-strategy associations. + */ + private Map initClassStrategyMap() { + Map classMapping = new HashMap<>(); + classMapping.put(Boolean.class, new BooleanStrategy()); + classMapping.put(Set.class, new SetStrategy()); + classMapping.put(Integer.class, new IntegerStrategy()); + classMapping.put(Long.class, new LongStrategy()); + classMapping.put(Float.class, new FloatStrategy()); + classMapping.put(Double.class, new DoubleStrategy()); + classMapping.put(Short.class, new ShortStrategy()); + classMapping.put(BigDecimal.class, new BigDecimalStrategy()); + classMapping.put(LocalDate.class, new DateStrategy()); + classMapping.put(LocalTime.class, new TimeStrategy()); + classMapping.put(LocalDateTime.class, new DateTimeStrategy()); + classMapping.put(GradoopId.class, new GradoopIdStrategy()); + classMapping.put(String.class, new StringStrategy()); + classMapping.put(List.class, new ListStrategy()); + classMapping.put(Map.class, new MapStrategy()); + + return Collections.unmodifiableMap(classMapping); + } + + /** + * Initializes byte-strategy mapping. + * + * @return Array containing PropertyValueStrategies with their respective type byte as index. + */ + private PropertyValueStrategy[] initByteStrategyMap() { + PropertyValueStrategy[] byteMapping = new PropertyValueStrategy[classStrategyMap.size() + 1]; + for (PropertyValueStrategy strategy : classStrategyMap.values()) { + byteMapping[strategy.getRawType()] = strategy; + } + byteMapping[nullStrategy.getRawType()] = nullStrategy; + + return byteMapping; + + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/PropertyValueStrategyUtils.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/PropertyValueStrategyUtils.java new file mode 100644 index 000000000000..229227353d87 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/PropertyValueStrategyUtils.java @@ -0,0 +1,259 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.gradoop.common.exceptions.UnsupportedTypeException; +import java.math.BigDecimal; +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Utility class which mainly handles comparing any values of the type {@code Numerical}. + */ +class PropertyValueStrategyUtils { + + /** + * Short type. + */ + private static final int SHORT = 0; + /** + * Integer type. + */ + private static final int INT = 1; + /** + * Long type. + */ + private static final int LONG = 2; + /** + * Float type. + */ + private static final int FLOAT = 3; + /** + * Double type. + */ + private static final int DOUBLE = 4; + /** + * Big decimal type. + */ + private static final int BIG_DECIMAL = 5; + + /** + * Compares a numerical to an object. The object needs to be an instance of {@code Number}. + * + * @param aValue first value + * @param other second value + * @return a negative integer, zero, or a positive integer as aValue is less than, equal to, or + * greater than bValue. + * @throws IllegalArgumentException if other is not an instance of number. + */ + static int compareNumerical(Number aValue, Object other) { + if (!(other instanceof Number)) { + throw new IllegalArgumentException(String.format( + "Incompatible types: %s, %s", aValue.getClass(), other.getClass())); + } + + Number bValue = (Number) other; + int aType = getType(aValue); + int bType = getType(bValue); + + boolean sameType = aType == bType; + + int maxType = Math.max(aType, bType); + + int result; + + if (maxType == SHORT) { + result = Short.compare(aValue.shortValue(), bValue.shortValue()); + + } else if (maxType == INT) { + int a; + int b; + + if (sameType) { + a = aValue.intValue(); + b = bValue.intValue(); + } else { + a = aType == INT ? aValue.intValue() : aValue.shortValue(); + b = bType == INT ? bValue.intValue() : bValue.shortValue(); + } + + result = Integer.compare(a, b); + + } else if (maxType == FLOAT) { + float a; + float b; + + if (sameType) { + a = aValue.floatValue(); + b = bValue.floatValue(); + } else { + a = aType == FLOAT ? aValue.floatValue() : floatValue(aValue, aType); + b = bType == FLOAT ? bValue.floatValue() : floatValue(bValue, bType); + } + + result = Float.compare(a, b); + + } else if (maxType == LONG) { + long a; + long b; + + if (sameType) { + a = aValue.longValue(); + b = bValue.longValue(); + } else { + a = aType == LONG ? aValue.longValue() : longValue(aValue, aType); + b = bType == LONG ? bValue.longValue() : longValue(bValue, bType); + } + + result = Long.compare(a, b); + + } else if (maxType == DOUBLE) { + double a; + double b; + + if (sameType) { + a = aValue.doubleValue(); + b = bValue.doubleValue(); + } else { + a = aType == DOUBLE ? aValue.doubleValue() : doubleValue(aValue, aType); + b = bType == DOUBLE ? bValue.doubleValue() : doubleValue(bValue, bType); + } + + result = Double.compare(a, b); + + } else { + BigDecimal a; + BigDecimal b; + + if (sameType) { + a = (BigDecimal) aValue; + b = (BigDecimal) bValue; + } else { + a = aType == BIG_DECIMAL ? (BigDecimal) aValue : + bigDecimalValue(aValue, aType); + b = bType == BIG_DECIMAL ? (BigDecimal) bValue : + bigDecimalValue(bValue, bType); + } + + result = a.compareTo(b); + } + + return result; + } + + /** + * Gets the number type of the provided object. + * + * @param value some number object. + * @return integer representation of the objects data type. + */ + private static int getType(Number value) { + checkNotNull(value); + + int type; + + if (value instanceof Short) { + type = SHORT; + } else if (value instanceof Integer) { + type = INT; + } else if (value instanceof Long) { + type = LONG; + } else if (value instanceof Float) { + type = FLOAT; + } else if (value instanceof Double) { + type = DOUBLE; + } else if (value instanceof BigDecimal) { + type = BIG_DECIMAL; + } else { + throw new UnsupportedTypeException(value.getClass()); + } + + return type; + } + + /** + * Converts a value of a lower domain numerical type to BigDecimal. + * + * @param value value + * @param type type + * @return converted value + */ + private static BigDecimal bigDecimalValue(Number value, int type) { + switch (type) { + case SHORT: + return BigDecimal.valueOf(value.shortValue()); + case INT: + return BigDecimal.valueOf(value.intValue()); + case LONG: + return BigDecimal.valueOf(value.longValue()); + case FLOAT: + return BigDecimal.valueOf(value.floatValue()); + default: + return BigDecimal.valueOf(value.doubleValue()); + } + } + + /** + * Converts a value of a lower domain numerical type to Double. + * + * @param value value + * @param type type + * @return converted value + */ + private static double doubleValue(Number value, int type) { + switch (type) { + case SHORT: + return value.shortValue(); + case INT: + return value.intValue(); + case LONG: + return value.longValue(); + default: + return value.floatValue(); + } + } + + /** + * Converts a value of a lower domain numerical type to Long. + * + * @param value value + * @param type type + * @return converted value + */ + private static long longValue(Number value, int type) { + if (type == SHORT) { + return value.shortValue(); + } + return value.intValue(); + } + + /** + * Converts a value of a lower domain numerical type to Float. + * + * @param value value + * @param type type + * @return converted value + */ + private static float floatValue(Number value, int type) { + switch (type) { + case SHORT: + return value.shortValue(); + case INT: + return value.intValue(); + default: + return value.longValue(); + } + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/SetStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/SetStrategy.java new file mode 100644 index 000000000000..3b34933f844d --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/SetStrategy.java @@ -0,0 +1,144 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code Set}. + */ +public class SetStrategy extends AbstractVariableSizedPropertyValueStrategy> { + + @Override + public Set read(DataInputView inputView, byte typeByte) throws IOException { + byte[] rawBytes = readVariableSizedData(inputView, typeByte); + + PropertyValue item; + Set set = new HashSet<>(); + DataInputViewStreamWrapper internalInputView = createInputView(rawBytes); + + try { + while (internalInputView.available() > 0) { + item = new PropertyValue(); + item.read(internalInputView); + + set.add(item); + } + } catch (IOException e) { + throw new IOException("Error reading PropertyValue with SetStrategy.", e); + } + + return set; + } + + @Override + public int compare(Set value, Object other) { + throw new UnsupportedOperationException("Method compare() is not supported for Set."); + } + + @Override + public boolean is(Object value) { + if (!(value instanceof Set)) { + return false; + } + for (Object item : (Set) value) { + // Set is not a valid property value if it contains any object + // that is not a property value itself. + if (!(item instanceof PropertyValue)) { + return false; + } + } + + return true; + } + + @Override + public Class> getType() { + return (Class) Set.class; + } + + /** + * {@inheritDoc} + * @throws IOException if converting the byte array to a Set fails. + */ + @Override + public Set get(byte[] bytes) throws IOException { + PropertyValue entry; + Set set = new HashSet<>(); + + try (ByteArrayInputStream byteStream = new ByteArrayInputStream(bytes); + DataInputStream inputStream = new DataInputStream(byteStream); + DataInputViewStreamWrapper internalInputView = + new DataInputViewStreamWrapper(inputStream)) { + + internalInputView.skipBytesToRead(1); + + while (inputStream.available() > 0) { + entry = new PropertyValue(); + entry.read(internalInputView); + + set.add(entry); + } + } catch (IOException e) { + throw new IOException("Error reading PropertyValue with SetStrategy.", e); + } + + return set; + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_SET; + } + + /** + * {@inheritDoc} + * @throws IOException if converting the value to a byte array fails. + */ + @Override + public byte[] getRawBytes(Set value) throws IOException { + int size = value.stream().mapToInt(PropertyValue::byteSize) + .sum() + PropertyValue.OFFSET + Bytes.SIZEOF_SHORT; + + try (ByteArrayOutputStream byteStream = new ByteArrayOutputStream(size); + DataOutputStream outputStream = new DataOutputStream(byteStream); + DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream)) { + + outputStream.write(getRawType()); + + for (PropertyValue entry : value) { + entry.write(outputView); + } + + return byteStream.toByteArray(); + } catch (IOException e) { + throw new IOException("Error writing PropertyValue with SetStrategy.", e); + } + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/ShortStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/ShortStrategy.java new file mode 100644 index 000000000000..611ef5414400 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/ShortStrategy.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code Short}. + */ +public class ShortStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public Short read(DataInputView inputView, byte typeByte) throws IOException { + int length = Bytes.SIZEOF_SHORT; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return Bytes.toShort(rawBytes); + } + + @Override + public int compare(Short value, Object other) { + return PropertyValueStrategyUtils.compareNumerical(value, other); + } + + @Override + public boolean is(Object value) { + return value instanceof Short; + } + + @Override + public Class getType() { + return Short.class; + } + + @Override + public Short get(byte[] bytes) { + return Bytes.toShort(bytes, PropertyValue.OFFSET); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_SHORT; + } + + @Override + public byte[] getRawBytes(Short value) { + byte[] rawBytes = new byte[PropertyValue.OFFSET + Bytes.SIZEOF_SHORT]; + rawBytes[0] = getRawType(); + Bytes.putShort(rawBytes, PropertyValue.OFFSET, value); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/StringStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/StringStrategy.java new file mode 100644 index 000000000000..60a3b8171466 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/StringStrategy.java @@ -0,0 +1,72 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code String}. + */ +public class StringStrategy extends AbstractVariableSizedPropertyValueStrategy { + + @Override + public String read(DataInputView inputView, byte typeByte) throws IOException { + byte[] rawBytes = readVariableSizedData(inputView, typeByte); + return Bytes.toString(rawBytes); + } + + @Override + public int compare(String value, Object other) { + if (other instanceof String) { + return value.compareTo((String) other); + } + throw new IllegalArgumentException(String.format( + "Incompatible types: %s, %s", value.getClass(), other.getClass())); + } + + @Override + public boolean is(Object value) { + return value instanceof String; + } + + @Override + public Class getType() { + return String.class; + } + + @Override + public String get(byte[] bytes) { + return Bytes.toString(bytes, PropertyValue.OFFSET, bytes.length - PropertyValue.OFFSET); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_STRING; + } + + @Override + public byte[] getRawBytes(String value) { + byte[] valueBytes = Bytes.toBytes(value); + byte[] rawBytes = new byte[PropertyValue.OFFSET + valueBytes.length]; + rawBytes[0] = getRawType(); + Bytes.putBytes(rawBytes, PropertyValue.OFFSET, valueBytes, 0, valueBytes.length); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/TimeStrategy.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/TimeStrategy.java new file mode 100644 index 000000000000..67494c83edb1 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/TimeStrategy.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.common.model.impl.properties.strategies; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.hadoop.hbase.util.Bytes; +import org.gradoop.common.model.impl.properties.DateTimeSerializer; +import org.gradoop.common.model.impl.properties.PropertyValue; +import java.io.IOException; +import java.time.LocalTime; +import java.util.Arrays; + +/** + * Strategy class for handling {@code PropertyValue} operations with a value of the type + * {@code LocalTime}. + */ +public class TimeStrategy extends AbstractFixSizedPropertyValueStrategy { + + @Override + public LocalTime read(DataInputView inputView, byte typeByte) throws IOException { + int length = DateTimeSerializer.SIZEOF_TIME; + byte[] rawBytes = new byte[length]; + + for (int i = 0; i < rawBytes.length; i++) { + rawBytes[i] = inputView.readByte(); + } + + return DateTimeSerializer.deserializeTime(rawBytes); + } + + @Override + public int compare(LocalTime value, Object other) { + if (other instanceof LocalTime) { + return value.compareTo((LocalTime) other); + } + throw new IllegalArgumentException(String.format( + "Incompatible types: %s, %s", value.getClass(), other.getClass())); + } + + @Override + public boolean is(Object value) { + return value instanceof LocalTime; + } + + @Override + public Class getType() { + return LocalTime.class; + } + + @Override + public LocalTime get(byte[] bytes) { + return DateTimeSerializer.deserializeTime( + Arrays.copyOfRange( + bytes, PropertyValue.OFFSET, DateTimeSerializer.SIZEOF_TIME + PropertyValue.OFFSET + )); + } + + @Override + public byte getRawType() { + return PropertyValue.TYPE_TIME; + } + + @Override + public byte[] getRawBytes(LocalTime value) { + byte[] valueBytes = DateTimeSerializer.serializeTime(value); + byte[] rawBytes = new byte[PropertyValue.OFFSET + DateTimeSerializer.SIZEOF_TIME]; + rawBytes[0] = getRawType(); + Bytes.putBytes(rawBytes, PropertyValue.OFFSET, valueBytes, 0, valueBytes.length); + return rawBytes; + } +} diff --git a/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/package-info.java b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/package-info.java new file mode 100644 index 000000000000..9d694305bd68 --- /dev/null +++ b/gradoop-common/src/main/java/org/gradoop/common/model/impl/properties/strategies/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains classes that implement the {@code PropertyValueStrategy} interface as well as their + * factory and utility classes. + */ +package org.gradoop.common.model.impl.properties.strategies; diff --git a/gradoop-common/src/main/java/org/gradoop/common/util/AsciiGraphLoader.java b/gradoop-common/src/main/java/org/gradoop/common/util/AsciiGraphLoader.java index f8f226ab805f..3e16dbc2ee9c 100644 --- a/gradoop-common/src/main/java/org/gradoop/common/util/AsciiGraphLoader.java +++ b/gradoop-common/src/main/java/org/gradoop/common/util/AsciiGraphLoader.java @@ -156,7 +156,7 @@ AsciiGraphLoader fromString(String asciiGraph, * @param EPGM edge type * * @return AsciiGraphLoader - * @throws IOException + * @throws IOException on failure */ public static @@ -180,7 +180,7 @@ AsciiGraphLoader fromFile(String fileName, * @param EPGM edge type * * @return AsciiGraphLoader - * @throws IOException + * @throws IOException on failure */ public static diff --git a/gradoop-common/src/main/java/org/gradoop/common/util/MathHelper.java b/gradoop-common/src/main/java/org/gradoop/common/util/MathHelper.java deleted file mode 100644 index 925cec8c52cf..000000000000 --- a/gradoop-common/src/main/java/org/gradoop/common/util/MathHelper.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright © 2014 - 2019 Leipzig University (Database Research Group) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gradoop.common.util; - -/** - * Contains math helper methods. - */ -public abstract class MathHelper { - /** - * Pairing function to uniquely encode two natural numbers into a single - * number. - * - * @param x first number (>= 0) - * @param y second number (>= 0) - * @return uniquely encoded natural number - */ - public static long cantor(long x, long y) { - return (x + y + 1) * (x + y) / 2 + y; - } - - /** - * Creates number pair from given cantor number. - * - * @param z cantor number - * @return first and second number - */ - public static long[] reverseCantor(long z) { - long[] pair = new long[2]; - long t = (long) (Math.floor(-1d + Math.sqrt(1d + 8 * z) / 2d)); - pair[0] = t * (t + 3) / 2 - z; - pair[1] = z - t * (t + 1) / 2; - return pair; - } -} diff --git a/gradoop-common/src/main/resources/META-INF/NOTICE b/gradoop-common/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..f8c2f341e183 --- /dev/null +++ b/gradoop-common/src/main/resources/META-INF/NOTICE @@ -0,0 +1,9 @@ +gradoop-common +Copyright 2014-2019 Leipzig University (Database Research Group) + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.github.s1ck:gdl:0.3 +- org.apache.flink:flink-java:1.7.0 +- org.apache.hbase:hbase-server:1.4.3 +- org.codehaus.jettison:jettison:1.3.7 diff --git a/gradoop-common/src/main/resources/META-INF/licenses/LICENSE.MongoDB b/gradoop-common/src/main/resources/META-INF/licenses/LICENSE.MongoDB new file mode 100644 index 000000000000..2f44352e7353 --- /dev/null +++ b/gradoop-common/src/main/resources/META-INF/licenses/LICENSE.MongoDB @@ -0,0 +1,87 @@ +1) Copyright 2008-2015 MongoDB, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +2) The following files: Immutable.java, NotThreadSafe.java, ThreadSafe.java + + Copyright (c) 2005 Brian Goetz and Tim Peierls + Released under the Creative Commons Attribution License (http://creativecommons.org/licenses/by/2.5) + Official home: http://www.jcip.net + + Any republication or derived work distributed in source code form + must include this copyright and license notice. + +3) The following files: Assertions.java, AbstractCopyOnWriteMap.java, CopyOnWriteMap.java + + Copyright (c) 2008-2014 Atlassian Pty Ltd + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +4) The following files: Beta.java + + Copyright 2010 The Guava Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +5) The following files: Base64Codec.java + + Copyright 1999,2005 The Apache Software Foundation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +6) The following files: ReadTimeoutHandler.java + + Copyright 2015 MongoDB, Inc. + Copyright 2012 The Netty Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/gradoop-common/src/test/java/org/gradoop/common/GradoopTestUtils.java b/gradoop-common/src/test/java/org/gradoop/common/GradoopTestUtils.java index 91bd59c67dbc..10a39b0a564d 100644 --- a/gradoop-common/src/test/java/org/gradoop/common/GradoopTestUtils.java +++ b/gradoop-common/src/test/java/org/gradoop/common/GradoopTestUtils.java @@ -172,6 +172,7 @@ public class GradoopTestUtils { * gradoop/dev-support/social-network.pdf * * @return graph store containing a simple social network for tests. + * @throws IOException on failure */ public static AsciiGraphLoader getSocialNetworkLoader() throws IOException { diff --git a/gradoop-common/src/test/java/org/gradoop/common/model/impl/id/GradoopIdsTest.java b/gradoop-common/src/test/java/org/gradoop/common/model/impl/id/GradoopIdsTest.java index 83cf22636655..5ae8f4acc560 100644 --- a/gradoop-common/src/test/java/org/gradoop/common/model/impl/id/GradoopIdsTest.java +++ b/gradoop-common/src/test/java/org/gradoop/common/model/impl/id/GradoopIdsTest.java @@ -34,7 +34,7 @@ public class GradoopIdsTest { @Test - public void testAdd() throws Exception { + public void testAdd() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); @@ -57,7 +57,7 @@ public void testAdd() throws Exception { } @Test - public void testContains() throws Exception { + public void testContains() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); @@ -70,7 +70,7 @@ public void testContains() throws Exception { } @Test - public void testAddAllCollection() throws Exception { + public void testAddAllCollection() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); GradoopId id3 = GradoopId.get(); @@ -88,7 +88,7 @@ public void testAddAllCollection() throws Exception { } @Test - public void testAddAllGradoopIds() throws Exception { + public void testAddAllGradoopIds() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); GradoopIdSet list1 = new GradoopIdSet(); @@ -107,7 +107,7 @@ public void testAddAllGradoopIds() throws Exception { } @Test - public void testContainsAllCollection() throws Exception { + public void testContainsAllCollection() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); GradoopId id3 = GradoopId.get(); @@ -123,7 +123,7 @@ public void testContainsAllCollection() throws Exception { } @Test - public void testContainsAllGradoopIds() throws Exception { + public void testContainsAllGradoopIds() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); GradoopId id3 = GradoopId.get(); @@ -140,7 +140,7 @@ public void testContainsAllGradoopIds() throws Exception { } @Test - public void testContainsAny() throws Exception { + public void testContainsAny() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); GradoopId id3 = GradoopId.get(); @@ -157,7 +157,7 @@ public void testContainsAny() throws Exception { } @Test - public void testContainsAny1() throws Exception { + public void testContainsAny1() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); GradoopId id3 = GradoopId.get(); @@ -173,7 +173,7 @@ public void testContainsAny1() throws Exception { } @Test - public void testIsEmpty() throws Exception { + public void testIsEmpty() { GradoopIdSet set1 = GradoopIdSet.fromExisting(GradoopId.get()); assertFalse(set1.isEmpty()); @@ -204,7 +204,7 @@ public void testWriteAndReadFields() throws Exception { } @Test - public void testIterator() throws Exception { + public void testIterator() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); @@ -220,7 +220,7 @@ public void testIterator() throws Exception { } @Test(expected = NoSuchElementException.class) - public void testIteratorException() throws Exception { + public void testIteratorException() { GradoopIdSet ids = new GradoopIdSet(); Iterator idsIterator = ids.iterator(); @@ -230,7 +230,7 @@ public void testIteratorException() throws Exception { } @Test - public void testClear() throws Exception { + public void testClear() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); @@ -246,7 +246,7 @@ public void testClear() throws Exception { } @Test - public void testSize() throws Exception { + public void testSize() { GradoopId id1 = GradoopId.get(); GradoopId id2 = GradoopId.get(); diff --git a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertiesTest.java b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertiesTest.java index eb129d7c07d5..65f9fe095e64 100644 --- a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertiesTest.java +++ b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertiesTest.java @@ -28,7 +28,7 @@ public class PropertiesTest { @Test - public void testCreateFromMap() throws Exception { + public void testCreateFromMap() { Properties properties = Properties.createFromMap(SUPPORTED_PROPERTIES); assertEquals(SUPPORTED_PROPERTIES.size(), properties.size()); @@ -41,7 +41,7 @@ public void testCreateFromMap() throws Exception { } @Test - public void testGetKeys() throws Exception { + public void testGetKeys() { Properties properties = Properties.createFromMap(SUPPORTED_PROPERTIES); List keyList = Lists.newArrayList(properties.getKeys()); @@ -53,7 +53,7 @@ public void testGetKeys() throws Exception { } @Test - public void testContainsKey() throws Exception { + public void testContainsKey() { Properties properties = Properties.create(); assertFalse("unexpected key found", properties.containsKey(KEY_1)); @@ -63,7 +63,7 @@ public void testContainsKey() throws Exception { } @Test - public void testGet() throws Exception { + public void testGet() { Properties properties = Properties.createFromMap(SUPPORTED_PROPERTIES); assertNotNull("property was null", properties.get(KEY_1)); @@ -72,7 +72,7 @@ public void testGet() throws Exception { } @Test - public void testSet() throws Exception { + public void testSet() { Properties properties = Properties.create(); properties.set(Property.create(KEY_1, BOOL_VAL_1)); @@ -84,7 +84,7 @@ public void testSet() throws Exception { } @Test - public void testSet1() throws Exception { + public void testSet1() { Properties properties = Properties.create(); properties.set(KEY_1, PropertyValue.create(BOOL_VAL_1)); @@ -96,7 +96,7 @@ public void testSet1() throws Exception { } @Test - public void testSet2() throws Exception { + public void testSet2() { Properties properties = Properties.create(); properties.set(KEY_1, BOOL_VAL_1); @@ -108,7 +108,7 @@ public void testSet2() throws Exception { } @Test - public void testRemove() throws Exception { + public void testRemove() { Properties properties = Properties.create(); PropertyValue removed; @@ -124,7 +124,7 @@ public void testRemove() throws Exception { } @Test - public void testRemove2() throws Exception { + public void testRemove2() { Properties properties = Properties.create(); PropertyValue removed; @@ -140,7 +140,7 @@ public void testRemove2() throws Exception { } @Test - public void testClear() throws Exception { + public void testClear() { Properties properties = Properties.create(); properties.set(KEY_1, BOOL_VAL_1); properties.clear(); @@ -148,7 +148,7 @@ public void testClear() throws Exception { } @Test - public void testSize() throws Exception { + public void testSize() { Properties properties = Properties.create(); assertEquals("wrong size", 0, properties.size()); properties.set(KEY_1, BOOL_VAL_1); @@ -161,7 +161,7 @@ public void testSize() throws Exception { } @Test - public void testIsEmpty() throws Exception { + public void testIsEmpty() { Properties properties = Properties.create(); assertTrue("properties was not empty", properties.isEmpty()); properties.set(KEY_1, BOOL_VAL_1); @@ -169,7 +169,7 @@ public void testIsEmpty() throws Exception { } @Test - public void testEqualsAndHashCode() throws Exception { + public void testEqualsAndHashCode() { Properties properties1 = Properties.createFromMap(SUPPORTED_PROPERTIES); Properties properties2 = Properties.createFromMap(SUPPORTED_PROPERTIES); Properties properties3 = Properties.createFromMap(SUPPORTED_PROPERTIES); @@ -194,7 +194,7 @@ public void testEqualsAndHashCode() throws Exception { } @Test - public void testIterator() throws Exception { + public void testIterator() { Properties properties = Properties.createFromMap(SUPPORTED_PROPERTIES); for (Property property : properties) { diff --git a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyTest.java b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyTest.java index 240f75362e4d..9d9d255c6827 100644 --- a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyTest.java +++ b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyTest.java @@ -23,13 +23,13 @@ public class PropertyTest { @Test - public void testGetKey() throws Exception { + public void testGetKey() { Property property = new Property("key", PropertyValue.create(10)); assertEquals("key", property.getKey()); } @Test - public void testSetKey() throws Exception { + public void testSetKey() { Property property = new Property("key", PropertyValue.create(10)); property.setKey("newKey"); assertEquals("newKey", property.getKey()); @@ -58,14 +58,14 @@ public void testSetKeyEmpty2() { } @Test - public void testGetValue() throws Exception { + public void testGetValue() { PropertyValue propertyValue = PropertyValue.create(10); Property p = new Property("key", propertyValue); assertEquals(propertyValue, p.getValue()); } @Test - public void testSetValue() throws Exception { + public void testSetValue() { PropertyValue propertyValue = PropertyValue.create(10); Property p = new Property("key", PropertyValue.create(11)); p.setValue(propertyValue); @@ -84,7 +84,7 @@ public void testSetValueNull2() { } @Test - public void testEqualsAndHashCode() throws Exception { + public void testEqualsAndHashCode() { Property p1 = new Property("key1", PropertyValue.create(10)); Property p2 = new Property("key1", PropertyValue.create(10)); Property p3 = new Property("key1", PropertyValue.create(11)); @@ -105,7 +105,7 @@ public void testEqualsAndHashCode() throws Exception { } @Test - public void testCompareTo() throws Exception { + public void testCompareTo() { Property p1 = new Property("key1", PropertyValue.create(10)); Property p2 = new Property("key1", PropertyValue.create(10)); Property p3 = new Property("key2", PropertyValue.create(10)); diff --git a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueListTest.java b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueListTest.java index 8a9b6b584a4c..6c140d48d0d6 100644 --- a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueListTest.java +++ b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueListTest.java @@ -120,7 +120,7 @@ public void testIterator() throws Exception { } @Test - public void testEmptyIterator() throws Exception { + public void testEmptyIterator() { PropertyValueList p = new PropertyValueList(); List expected = Lists.newArrayList(p); diff --git a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueTest.java b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueTest.java index 0672f56313a5..a97ec8a0a559 100644 --- a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueTest.java +++ b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueTest.java @@ -16,11 +16,11 @@ package org.gradoop.common.model.impl.properties; import com.google.common.collect.Lists; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; import org.gradoop.common.model.impl.id.GradoopId; import org.gradoop.common.exceptions.UnsupportedTypeException; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.io.IOException; import java.math.BigDecimal; @@ -43,57 +43,11 @@ public class PropertyValueTest { - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Test - public void testBigDecimalConversion() { - PropertyValue property; - BigDecimal decimalValue; - - // SHORT - property = create(SHORT_VAL_e); - decimalValue = BigDecimal.valueOf(SHORT_VAL_e); - assertEquals(decimalValue, property.getBigDecimal()); - - // INT - property = create(INT_VAL_2); - decimalValue = BigDecimal.valueOf(INT_VAL_2); - assertEquals(decimalValue, property.getBigDecimal()); - - // LONG - property = create(LONG_VAL_3); - decimalValue = BigDecimal.valueOf(LONG_VAL_3); - assertEquals(decimalValue, property.getBigDecimal()); - - // FLOAT - property = create(FLOAT_VAL_4); - decimalValue = BigDecimal.valueOf(FLOAT_VAL_4); - assertEquals(decimalValue, property.getBigDecimal()); - - // DOUBLE - property = create(DOUBLE_VAL_5); - decimalValue = BigDecimal.valueOf(DOUBLE_VAL_5); - assertEquals(decimalValue, property.getBigDecimal()); - - // STRING - property = create("-3.5"); - decimalValue = new BigDecimal("-3.5"); - assertEquals(decimalValue, property.getBigDecimal()); - - exception.expect(NumberFormatException.class); - property = create("No Number"); - property.getBigDecimal(); - } - - @Test - public void testFailedBigDecimalConversion() { - exception.expect(ClassCastException.class); - create(BOOL_VAL_1).getBigDecimal(); - } - + /** + * Tests if {@link PropertyValue#create(Object)} works with supported types. + */ @Test - public void testCreate() throws Exception { + public void testCreate() { // null PropertyValue p = create(null); assertTrue(p.isNull()); @@ -164,23 +118,162 @@ public void testCreate() throws Exception { * Test copying the property value */ @Test - public void testCopy() { - // copy primitive value + public void testCopyBoolean() { PropertyValue p = create(BOOL_VAL_1); PropertyValue copy = p.copy(); assertEquals(p, copy); assertNotSame(p, copy); + } - // deep copy complex value - p = create(LIST_VAL_a); - copy = p.copy(); + /** + * Tests if {@link PropertyValue#copy()} works when a float type is wrapped. + */ + @Test + public void testCopyFloat() { + PropertyValue p = create(FLOAT_VAL_4); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(FLOAT_VAL_4, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a double type is wrapped. + */ + @Test + public void testCopyDouble() { + PropertyValue p = create(DOUBLE_VAL_5); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(DOUBLE_VAL_5, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a {@link BigDecimal} type is wrapped. + */ + @Test + public void testCopyBigDecimal() { + PropertyValue p = create(BIG_DECIMAL_VAL_7); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(BIG_DECIMAL_VAL_7, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a {@link GradoopId} type is wrapped. + */ + @Test + public void testCopyGradoopId() { + PropertyValue p = create(GRADOOP_ID_VAL_8); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(GRADOOP_ID_VAL_8, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a {@link LocalDate} type is wrapped. + */ + @Test + public void testCopyDate() { + PropertyValue p = create(DATE_VAL_b); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(DATE_VAL_b, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a {@link LocalDateTime} type is wrapped. + */ + @Test + public void testCopyDateTime() { + PropertyValue p = create(DATETIME_VAL_d); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(DATETIME_VAL_d, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a {@link LocalTime} type is wrapped. + */ + @Test + public void testCopyTime() { + PropertyValue p = create(TIME_VAL_c); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(TIME_VAL_c, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when an int type is wrapped. + */ + @Test + public void testCopyInteger() { + PropertyValue p = create(INT_VAL_2); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a {@link String} type is wrapped. + */ + @Test + public void testCopyString() { + PropertyValue p = create(STRING_VAL_6); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(STRING_VAL_6, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a {@link Map} type is wrapped. + */ + @Test + public void testCopyMap() { + PropertyValue p = create(MAP_VAL_9); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(MAP_VAL_9, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue#copy()} works when a {@link List} type is wrapped. + */ + @Test + public void testCopyList() { + PropertyValue p = create(LIST_VAL_a); + PropertyValue copy = p.copy(); assertEquals(p, copy); assertNotSame(p, copy); assertNotSame(LIST_VAL_a, copy.getObject()); } + /** + * Tests if {@link PropertyValue#copy()} works when a {@link Set} type is wrapped. + */ @Test - public void testSetAndGetObject() throws Exception { + public void testCopySet() { + PropertyValue p = create(SET_VAL_f); + PropertyValue copy = p.copy(); + assertEquals(p, copy); + assertNotSame(p, copy); + assertNotSame(SET_VAL_f, copy.getObject()); + } + + /** + * Tests if {@link PropertyValue} setter and getter methods work as expected. + * X -> p.setObject(X) -> p.getObject() -> X | where X equals X + */ + @Test + public void testSetAndGetObject() { PropertyValue p = new PropertyValue(); // null p.setObject(null); @@ -248,14 +341,21 @@ public void testSetAndGetObject() throws Exception { assertEquals(SET_VAL_f, p.getSet()); } + /** + * Tests if {@link PropertyValue#setObject(Object)} throws an {@link UnsupportedTypeException} if + * an unsupported type is passed as an argument. + */ @Test(expected = UnsupportedTypeException.class) public void testSetObjectWithUnsupportedType() { PropertyValue p = new PropertyValue(); p.setObject(new PriorityQueue<>()); } + /** + * Tests whether {@link PropertyValue#isNull()} returns true iff the instance wraps {@code null}. + */ @Test - public void testIsNull() throws Exception { + public void testIsNull() { PropertyValue p = PropertyValue.create(null); assertTrue(p.isNull()); assertFalse(p.isBoolean()); @@ -275,8 +375,12 @@ public void testIsNull() throws Exception { assertFalse(p.isSet()); } + /** + * Tests whether {@link PropertyValue#isBoolean()} returns true iff the instance wraps a + * {@code boolean}. + */ @Test - public void testIsBoolean() throws Exception { + public void testIsBoolean() { PropertyValue p = PropertyValue.create(true); assertFalse(p.isNull()); assertTrue(p.isBoolean()); @@ -296,21 +400,30 @@ public void testIsBoolean() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getBoolean()}. + */ @Test - public void testGetBoolean() throws Exception { + public void testGetBoolean() { PropertyValue p = PropertyValue.create(BOOL_VAL_1); assertEquals(BOOL_VAL_1, p.getBoolean()); } + /** + * Tests {@link PropertyValue#setBoolean(boolean)}. + */ @Test - public void testSetBoolean() throws Exception { + public void testSetBoolean() { PropertyValue p = new PropertyValue(); p.setBoolean(BOOL_VAL_1); assertEquals(BOOL_VAL_1, p.getBoolean()); } + /** + * Tests {@link PropertyValue#isShort()}. + */ @Test - public void testIsShort() throws Exception { + public void testIsShort() { PropertyValue p = PropertyValue.create(SHORT_VAL_e); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -330,21 +443,30 @@ public void testIsShort() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getShort()}. + */ @Test - public void testGetShort() throws Exception { + public void testGetShort() { PropertyValue p = PropertyValue.create(SHORT_VAL_e); assertEquals(SHORT_VAL_e, p.getShort()); } + /** + * Tests {@link PropertyValue#setShort(short)}. + */ @Test - public void testSetShort() throws Exception { + public void testSetShort() { PropertyValue p = new PropertyValue(); p.setShort(SHORT_VAL_e); assertEquals(SHORT_VAL_e, p.getShort()); } + /** + * Tests {@link PropertyValue#isInt()}. + */ @Test - public void testIsInt() throws Exception { + public void testIsInt() { PropertyValue p = PropertyValue.create(INT_VAL_2); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -364,21 +486,30 @@ public void testIsInt() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getInt()}. + */ @Test - public void testGetInt() throws Exception { + public void testGetInt() { PropertyValue p = PropertyValue.create(INT_VAL_2); assertEquals(INT_VAL_2, p.getInt()); } + /** + * Tests {@link PropertyValue#setInt(int)}. + */ @Test - public void testSetInt() throws Exception { + public void testSetInt() { PropertyValue p = new PropertyValue(); p.setInt(INT_VAL_2); assertEquals(INT_VAL_2, p.getInt()); } + /** + * Tests {@link PropertyValue#isLong()}. + */ @Test - public void testIsLong() throws Exception { + public void testIsLong() { PropertyValue p = PropertyValue.create(LONG_VAL_3); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -398,21 +529,30 @@ public void testIsLong() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getLong()}. + */ @Test - public void testGetLong() throws Exception { + public void testGetLong() { PropertyValue p = PropertyValue.create(LONG_VAL_3); assertEquals(LONG_VAL_3, p.getLong()); } + /** + * Tests {@link PropertyValue#setLong(long)}. + */ @Test - public void testSetLong() throws Exception { + public void testSetLong() { PropertyValue p = new PropertyValue(); p.setLong(LONG_VAL_3); assertEquals(LONG_VAL_3, p.getLong()); } + /** + * Tests {@link PropertyValue#isFloat()}. + */ @Test - public void testIsFloat() throws Exception { + public void testIsFloat() { PropertyValue p = PropertyValue.create(FLOAT_VAL_4); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -432,21 +572,30 @@ public void testIsFloat() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getFloat()}. + */ @Test - public void testGetFloat() throws Exception { + public void testGetFloat() { PropertyValue p = PropertyValue.create(FLOAT_VAL_4); assertEquals(FLOAT_VAL_4, p.getFloat(), 0); } + /** + * Tests {@link PropertyValue#setFloat(float)}. + */ @Test - public void testSetFloat() throws Exception { + public void testSetFloat() { PropertyValue p = new PropertyValue(); p.setFloat(FLOAT_VAL_4); assertEquals(FLOAT_VAL_4, p.getFloat(), 0); } + /** + * Tests {@link PropertyValue#isDouble()}. + */ @Test - public void testIsDouble() throws Exception { + public void testIsDouble() { PropertyValue p = PropertyValue.create(DOUBLE_VAL_5); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -466,21 +615,30 @@ public void testIsDouble() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getDouble()}. + */ @Test - public void testGetDouble() throws Exception { + public void testGetDouble() { PropertyValue p = PropertyValue.create(DOUBLE_VAL_5); assertEquals(DOUBLE_VAL_5, p.getDouble(), 0); } + /** + * Tests {@link PropertyValue#setDouble(double)}. + */ @Test - public void testSetDouble() throws Exception { + public void testSetDouble() { PropertyValue p = new PropertyValue(); p.setDouble(DOUBLE_VAL_5); assertEquals(DOUBLE_VAL_5, p.getDouble(), 0); } + /** + * Tests {@link PropertyValue#isString()}. + */ @Test - public void testIsString() throws Exception { + public void testIsString() { PropertyValue p = PropertyValue.create(STRING_VAL_6); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -500,21 +658,30 @@ public void testIsString() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getString()}. + */ @Test - public void testGetString() throws Exception { + public void testGetString() { PropertyValue p = PropertyValue.create(STRING_VAL_6); assertEquals(STRING_VAL_6, p.getString()); } + /** + * Tests {@link PropertyValue#setString(String)}. + */ @Test - public void testSetString() throws Exception { + public void testSetString() { PropertyValue p = new PropertyValue(); p.setString(STRING_VAL_6); assertEquals(STRING_VAL_6, p.getString()); } + /** + * Tests {@link PropertyValue#isBigDecimal()}. + */ @Test - public void testIsBigDecimal() throws Exception { + public void testIsBigDecimal() { PropertyValue p = PropertyValue.create(BIG_DECIMAL_VAL_7); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -534,21 +701,30 @@ public void testIsBigDecimal() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getBigDecimal()}. + */ @Test - public void testGetBigDecimal() throws Exception { + public void testGetBigDecimal() { PropertyValue p = PropertyValue.create(BIG_DECIMAL_VAL_7); assertEquals(BIG_DECIMAL_VAL_7, p.getBigDecimal()); } + /** + * Tests {@link PropertyValue#setBigDecimal(BigDecimal)}. + */ @Test - public void testSetBigDecimal() throws Exception { + public void testSetBigDecimal() { PropertyValue p = new PropertyValue(); p.setBigDecimal(BIG_DECIMAL_VAL_7); assertEquals(BIG_DECIMAL_VAL_7, p.getBigDecimal()); } + /** + * Tests {@link PropertyValue#isGradoopId()}. + */ @Test - public void testIsGradoopId() throws Exception { + public void testIsGradoopId() { PropertyValue p = PropertyValue.create(GRADOOP_ID_VAL_8); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -568,21 +744,30 @@ public void testIsGradoopId() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getGradoopId()}. + */ @Test - public void testGetGradoopId() throws Exception { + public void testGetGradoopId() { PropertyValue p = PropertyValue.create(GRADOOP_ID_VAL_8); assertEquals(GRADOOP_ID_VAL_8, p.getGradoopId()); } + /** + * Tests {@link PropertyValue#setGradoopId(GradoopId)}. + */ @Test - public void testSetGradoopId() throws Exception { + public void testSetGradoopId() { PropertyValue p = new PropertyValue(); p.setGradoopId(GRADOOP_ID_VAL_8); assertEquals(GRADOOP_ID_VAL_8, p.getGradoopId()); } + /** + * Tests {@link PropertyValue#isMap()}. + */ @Test - public void testIsMap() throws Exception { + public void testIsMap() { PropertyValue p = PropertyValue.create(MAP_VAL_9); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -602,21 +787,30 @@ public void testIsMap() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getMap()}. + */ @Test - public void testGetMap() throws Exception { + public void testGetMap() { PropertyValue p = PropertyValue.create(MAP_VAL_9); assertEquals(MAP_VAL_9, p.getMap()); } + /** + * Tests {@link PropertyValue#setMap(Map)}. + */ @Test - public void testSetMap() throws Exception { + public void testSetMap() { PropertyValue p = new PropertyValue(); p.setMap(MAP_VAL_9); assertEquals(MAP_VAL_9, p.getMap()); } + /** + * Tests {@link PropertyValue#isList()}. + */ @Test - public void testIsList() throws Exception { + public void testIsList() { PropertyValue p = PropertyValue.create(LIST_VAL_a); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -636,21 +830,79 @@ public void testIsList() throws Exception { assertFalse(p.isSet()); } + /** + * Tests whether passing a {@link List} which is not parametrized as {@link PropertyValue} to + * {@link PropertyValue#create(Object)} will result in an {@link UnsupportedTypeException}. + */ + @Test(expected = UnsupportedTypeException.class) + public void testCreateWrongParameterizedList() { + List list = new ArrayList<>(); + list.add("test1"); + list.add("test2"); + list.add("test3"); + PropertyValue p = PropertyValue.create(list); + } + + /** + * Tests whether passing a {@link Map} which is not parametrized as {@link PropertyValue} to + * {@link PropertyValue#create(Object)} will result in an {@link UnsupportedTypeException}. + */ + @Test(expected = UnsupportedTypeException.class) + public void testCreateWrongParameterizedMap() { + Map map = new HashMap<>(); + map.put("key1", "val1"); + map.put("key2", "val2"); + map.put("key3", "val3"); + PropertyValue p = PropertyValue.create(map); + } + + /** + * Tests whether passing a {@link Set} which is not parametrized as {@link PropertyValue} to + * {@link PropertyValue#create(Object)} will result in an {@link UnsupportedTypeException}. + */ + @Test(expected = UnsupportedTypeException.class) + public void testCreateWrongParameterizedSet() { + Set set = new HashSet<>(); + set.add("test1"); + set.add("test2"); + set.add("test3"); + PropertyValue p = PropertyValue.create(set); + } + + /** + * Tests whether {@link PropertyValue#create(Object)} works when an empty {@link List} is + * provided. + */ @Test - public void testGetList() throws Exception { + public void testCreateEmptyList() { + List list = new ArrayList<>(); + PropertyValue p = PropertyValue.create(list); + } + + /** + * Tests {@link PropertyValue#getList()}. + */ + @Test + public void testGetList() { PropertyValue p = PropertyValue.create(LIST_VAL_a); assertEquals(LIST_VAL_a, p.getList()); } + /** + * Tests {@link PropertyValue#setList(List)}. + */ @Test - public void testSetList() throws Exception { + public void testSetList() { PropertyValue p = new PropertyValue(); p.setList(LIST_VAL_a); assertEquals(LIST_VAL_a, p.getList()); } + /** + * Tests {@link PropertyValue#isDate()}. + */ @Test - public void testIsDate() throws Exception { + public void testIsDate() { PropertyValue p = PropertyValue.create(DATE_VAL_b); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -670,21 +922,30 @@ public void testIsDate() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getDate()}. + */ @Test - public void testGetDate() throws Exception { + public void testGetDate() { PropertyValue p = PropertyValue.create(DATE_VAL_b); assertEquals(DATE_VAL_b, p.getDate()); } + /** + * Tests {@link PropertyValue#setDate(LocalDate)}. + */ @Test - public void testSetDate() throws Exception { + public void testSetDate() { PropertyValue p = new PropertyValue(); p.setDate(DATE_VAL_b); assertEquals(DATE_VAL_b, p.getDate()); } + /** + * Tests {@link PropertyValue#isTime()}. + */ @Test - public void testIsTime() throws Exception { + public void testIsTime() { PropertyValue p = PropertyValue.create(TIME_VAL_c); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -704,21 +965,30 @@ public void testIsTime() throws Exception { assertFalse(p.isSet()); } + /** + * tests {@link PropertyValue#getTime()}. + */ @Test - public void testGetTime() throws Exception { + public void testGetTime() { PropertyValue p = PropertyValue.create(TIME_VAL_c); assertEquals(TIME_VAL_c, p.getTime()); } + /** + * Tests {@link PropertyValue#setTime(LocalTime)}. + */ @Test - public void testSetTime() throws Exception { + public void testSetTime() { PropertyValue p = new PropertyValue(); p.setTime(TIME_VAL_c); assertEquals(TIME_VAL_c, p.getTime()); } + /** + * Tests {@link PropertyValue#isDateTime()}. + */ @Test - public void testIsDateTime() throws Exception { + public void testIsDateTime() { PropertyValue p = PropertyValue.create(DATETIME_VAL_d); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -738,21 +1008,30 @@ public void testIsDateTime() throws Exception { assertFalse(p.isSet()); } + /** + * Tests {@link PropertyValue#getDateTime()}. + */ @Test - public void testGetDateTime() throws Exception { + public void testGetDateTime() { PropertyValue p = PropertyValue.create(DATETIME_VAL_d); assertEquals(DATETIME_VAL_d, p.getDateTime()); } + /** + * Tests {@link PropertyValue#setDateTime(LocalDateTime)}. + */ @Test - public void testSetDateTime() throws Exception { + public void testSetDateTime() { PropertyValue p = new PropertyValue(); p.setDateTime(DATETIME_VAL_d); assertEquals(DATETIME_VAL_d, p.getDateTime()); } + /** + * Tests {@link PropertyValue#isSet()}. + */ @Test - public void testIsSet() throws Exception { + public void testIsSet() { PropertyValue p = PropertyValue.create(SET_VAL_f); assertFalse(p.isNull()); assertFalse(p.isBoolean()); @@ -772,21 +1051,30 @@ public void testIsSet() throws Exception { assertTrue(p.isSet()); } + /** + * Tests {@link PropertyValue#getSet()}. + */ @Test - public void testGetSet() throws Exception { + public void testGetSet() { PropertyValue p = PropertyValue.create(SET_VAL_f); assertEquals(SET_VAL_f, p.getSet()); } + /** + * Tests {@link PropertyValue#setSet(Set)}. + */ @Test - public void testSetSet() throws Exception { + public void testSetSet() { PropertyValue p = new PropertyValue(); p.setSet(SET_VAL_f); assertEquals(SET_VAL_f, p.getSet()); } + /** + * Tests {@link PropertyValue#isNumber()}. + */ @Test - public void testIsNumber() throws Exception { + public void testIsNumber() { PropertyValue p = PropertyValue.create(SHORT_VAL_e); assertTrue(p.isNumber()); p = PropertyValue.create(INT_VAL_2); @@ -822,8 +1110,11 @@ public void testIsNumber() throws Exception { assertFalse(p.isNumber()); } + /** + * Tests {@link PropertyValue#equals(Object)} and {@link PropertyValue#hashCode()}. + */ @Test - public void testEqualsAndHashCode() throws Exception { + public void testEqualsAndHashCode() { validateEqualsAndHashCode(create(null), create(null), create(false)); validateEqualsAndHashCode(create(true), create(true), create(false)); @@ -896,8 +1187,7 @@ public void testEqualsAndHashCode() throws Exception { validateEqualsAndHashCode(create(set1), create(set2), create(set3)); } - private void validateEqualsAndHashCode(PropertyValue p1, PropertyValue p2, - PropertyValue p3) { + private void validateEqualsAndHashCode(PropertyValue p1, PropertyValue p2, PropertyValue p3) { assertEquals(p1, p1); assertEquals(p1, p2); assertNotEquals(p1, p3); @@ -907,8 +1197,21 @@ private void validateEqualsAndHashCode(PropertyValue p1, PropertyValue p2, assertNotEquals(p1.hashCode(), p3.hashCode()); } + /** + * Tests whether an instance of {@link PropertyValue} which was created with {@code null} equals + * {@code null}. + */ @Test - public void testCompareTo() throws Exception { + public void testEqualsWithNull() { + PropertyValue p = PropertyValue.create(null); + assertNotEquals(p, null); + } + + /** + * Tests {@link PropertyValue#compareTo(PropertyValue)}. + */ + @Test + public void testCompareTo() { // null assertEquals(create(null).compareTo(create(null)), 0); // boolean @@ -1019,48 +1322,79 @@ public void testCompareTo() throws Exception { ); } + /** + * Tests whether {@link PropertyValue#compareTo(PropertyValue)} throws an + * {@link IllegalArgumentException} when the instances types are incomparable. + */ @Test(expected = IllegalArgumentException.class) public void testCompareToWithIncompatibleTypes() { create(10).compareTo(create("10")); } + /** + * Tests whether {@link PropertyValue#compareTo(PropertyValue)} throws an + * {@link IllegalArgumentException} if the instance is of {@link Map}. + */ @Test(expected = UnsupportedOperationException.class) public void testCompareToWithMap() { create(MAP_VAL_9).compareTo(create(MAP_VAL_9)); } + /** + * Tests whether {@link PropertyValue#compareTo(PropertyValue)} throws an + * {@link IllegalArgumentException} if the instance is of {@link List}. + */ @Test(expected = UnsupportedOperationException.class) public void testCompareToWithList() { create(LIST_VAL_a).compareTo(create(LIST_VAL_a)); } + /** + * Tests whether {@link PropertyValue#compareTo(PropertyValue)} throws an + * {@link IllegalArgumentException} if the instance is of {@link Set}. + */ @Test(expected = UnsupportedOperationException.class) public void testCompareToWithSet() { create(SET_VAL_f).compareTo(create(SET_VAL_f)); } + /** + * Tests {@link PropertyValue#setBytes(byte[])}. + */ @Test public void testArrayValueMaxSize() { PropertyValue property = new PropertyValue(); property.setBytes(new byte[PropertyValue.LARGE_PROPERTY_THRESHOLD]); } + /** + * Tests {@link PropertyValue#setBytes(byte[])} with value greater than {@link Short#MAX_VALUE}. + */ @Test public void testLargeArrayValue() { PropertyValue property = new PropertyValue(); property.setBytes(new byte[PropertyValue.LARGE_PROPERTY_THRESHOLD + 1]); } + /** + * Tests {@link PropertyValue#create(Object)} with large string. + */ @Test public void testStringValueMaxSize() { create(new String(new byte[PropertyValue.LARGE_PROPERTY_THRESHOLD])); } + /** + * Tests {@link PropertyValue#create(Object)} with string larger than {@link Short#MAX_VALUE}. + */ @Test public void testLargeString() { create(new String(new byte[PropertyValue.LARGE_PROPERTY_THRESHOLD + 10])); } + /** + * Tests {@link PropertyValue#create(Object)} with big {@link List}. + */ @Test public void testListValueMaxSize() { int n = PropertyValue.LARGE_PROPERTY_THRESHOLD / 9; @@ -1071,6 +1405,10 @@ public void testListValueMaxSize() { create(list); } + /** + * Tests {@link PropertyValue#create(Object)} with {@link List} of + * length > {@link Short#MAX_VALUE}. + */ @Test public void testLargeListValue() { // 8 bytes per double + 1 byte overhead @@ -1082,6 +1420,9 @@ public void testLargeListValue() { create(list); } + /** + * Tests {@link PropertyValue#create(Object)} with big {@link Map}. + */ @Test public void testMapValueMaxSize() { Map m = new HashMap<>(); @@ -1093,6 +1434,10 @@ public void testMapValueMaxSize() { create(m); } + /** + * Tests {@link PropertyValue#create(Object)} with {@link Map} of + * size > {@link Short#MAX_VALUE}. + */ @Test public void testLargeMapValue() { Map m = new HashMap<>(); @@ -1104,6 +1449,9 @@ public void testLargeMapValue() { create(m); } + /** + * Tests {@link PropertyValue#create(Object)} with big {@link Set}. + */ @Test public void testSetValueMaxSize() { Set s = new HashSet<>(); @@ -1115,6 +1463,10 @@ public void testSetValueMaxSize() { create(s); } + /** + * Tests {@link PropertyValue#create(Object)} with {@link Set} of + * size > {@link Short#MAX_VALUE}. + */ @Test public void testLargeSetValue() { Set s = new HashSet<>(); @@ -1126,6 +1478,9 @@ public void testLargeSetValue() { create(s); } + /** + * Tests {@link PropertyValue#create} with big {@link BigDecimal}. + */ @Test public void testBigDecimalValueMaxSize() { // internal representation of BigInteger needs 5 bytes @@ -1134,6 +1489,9 @@ public void testBigDecimalValueMaxSize() { create(new BigDecimal(new BigInteger(bigendian))); } + /** + * Tests {@link PropertyValue#create} large {@link BigDecimal}. + */ @Test public void testLargeBigDecimal() { byte [] bigendian = new byte[Short.MAX_VALUE + 10]; @@ -1141,6 +1499,12 @@ public void testLargeBigDecimal() { create(new BigDecimal(new BigInteger(bigendian))); } + /** + * Tests {@link PropertyValue#write(DataOutputView)} and + * {@link PropertyValue#read(DataInputView)}. + * + * @throws IOException if something goes wrong. + */ @Test public void testWriteAndReadFields() throws IOException { PropertyValue p = create(NULL_VAL_0); @@ -1192,6 +1556,53 @@ public void testWriteAndReadFields() throws IOException { assertEquals(p, writeAndReadFields(PropertyValue.class, p)); } + /** + * Tests that {@link PropertyValue#read(DataInputView)} and + * {@link PropertyValue#write(DataOutputView)} can handle large {@link String} values. + * + * @throws Exception when something goes wrong. + */ + @Test + public void testReadAndWriteLargeString() throws Exception { + PropertyValue p = create(new String(new byte[PropertyValue.LARGE_PROPERTY_THRESHOLD])); + assertEquals(p, writeAndReadFields(PropertyValue.class, p)); + } + + /** + * Tests that {@link PropertyValue#read(DataInputView)} and + * {@link PropertyValue#write(DataOutputView)} can handle large {@link BigDecimal} values. + * + * @throws Exception when something goes wrong. + */ + @Test + public void testReadAndWriteLargeBigDecimal() throws Exception { + byte [] bigEndian = new byte[Short.MAX_VALUE + 10]; + Arrays.fill(bigEndian, (byte) 121); + PropertyValue p = create(new BigDecimal(new BigInteger(bigEndian))); + assertEquals(p, writeAndReadFields(PropertyValue.class, p)); + } + + /** + * Tests that {@link PropertyValue#read(DataInputView)} and + * {@link PropertyValue#write(DataOutputView)} can handle large {@link Map} values. + * + * @throws Exception when something goes wrong. + */ + @Test + public void testReadAndWriteLargeMap() throws Exception { + HashMap largeMap = new HashMap<>(); + long neededEntries = PropertyValue.LARGE_PROPERTY_THRESHOLD / 10; + for (int i = 0; i < neededEntries; i++) { + largeMap.put(PropertyValue.create("key" + i), PropertyValue.create("value" + i)); + } + + PropertyValue p = create(largeMap); + assertEquals(p, writeAndReadFields(PropertyValue.class, p)); + } + + /** + * Tests {@link PropertyValue#getType()}. + */ @Test public void testGetType() { PropertyValue p = create(NULL_VAL_0); @@ -1246,11 +1657,10 @@ public void testGetType() { /** * Assumes that p1 == p2 < p3 */ - private void validateCompareTo(PropertyValue p1, PropertyValue p2, - PropertyValue p3) { - assertTrue(p1.compareTo(p1) == 0); - assertTrue(p1.compareTo(p2) == 0); - assertTrue(p2.compareTo(p1) == 0); + private void validateCompareTo(PropertyValue p1, PropertyValue p2, PropertyValue p3) { + assertEquals(0, p1.compareTo(p1)); + assertEquals(0, p1.compareTo(p2)); + assertEquals(0, p2.compareTo(p1)); assertTrue(p1.compareTo(p3) < 0); assertTrue(p3.compareTo(p1) > 0); assertTrue(p3.compareTo(p2) > 0); diff --git a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueUtilsTest.java b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueUtilsTest.java index 95d4c550c652..e05509e0bb00 100644 --- a/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueUtilsTest.java +++ b/gradoop-common/src/test/java/org/gradoop/common/model/impl/properties/PropertyValueUtilsTest.java @@ -70,8 +70,9 @@ public void testOr() { * Test class of {@link PropertyValueUtils.Numeric} */ public static class NumericTest { + @Test - public void testAddReturningBigDecimal() throws Exception { + public void testAddReturningBigDecimal() { PropertyValue p; // BigDecimal @@ -244,7 +245,7 @@ public void testAddReturningInteger() { } @Test - public void testMultiplyReturningBigDecimal() throws Exception { + public void testMultiplyReturningBigDecimal() { PropertyValue p; // BigDecimal @@ -417,7 +418,7 @@ public void testMultiplyReturningInteger() { } @Test - public void testMin() throws Exception { + public void testMin() { PropertyValue p; BigDecimal minBigDecimal = new BigDecimal("10"); BigDecimal maxBigDecimal = new BigDecimal("11"); @@ -692,7 +693,7 @@ public void testMin() throws Exception { } @Test - public void testMax() throws Exception { + public void testMax() { PropertyValue p; BigDecimal minBigDecimal = new BigDecimal("10"); BigDecimal maxBigDecimal = new BigDecimal("11"); diff --git a/gradoop-common/src/test/java/org/gradoop/common/util/AsciiGraphLoaderTest.java b/gradoop-common/src/test/java/org/gradoop/common/util/AsciiGraphLoaderTest.java index 7a7fba918930..962baecbb42d 100644 --- a/gradoop-common/src/test/java/org/gradoop/common/util/AsciiGraphLoaderTest.java +++ b/gradoop-common/src/test/java/org/gradoop/common/util/AsciiGraphLoaderTest.java @@ -53,7 +53,7 @@ public void testFromFile() throws Exception { } @Test - public void testGetGraphHeads() throws Exception { + public void testGetGraphHeads() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("[()]", config); @@ -68,7 +68,7 @@ public void testGetGraphHeads() throws Exception { } @Test - public void testGetGraphHeadByVariable() throws Exception { + public void testGetGraphHeadByVariable() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("g[()];h[()]", config); @@ -83,7 +83,7 @@ public void testGetGraphHeadByVariable() throws Exception { } @Test - public void testGetGraphHeadsByVariables() throws Exception { + public void testGetGraphHeadsByVariables() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("g[()],h[()]", config); @@ -94,7 +94,7 @@ public void testGetGraphHeadsByVariables() throws Exception { } @Test - public void testGetVertices() throws Exception { + public void testGetVertices() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("[()]", config); @@ -122,7 +122,7 @@ public void testGetVertexByVariable() { } @Test - public void testGetVerticesByVariables() throws Exception { + public void testGetVerticesByVariables() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("[(a),(b),(a)]", config); @@ -141,7 +141,7 @@ public void testGetVerticesByVariables() throws Exception { } @Test - public void testGetVerticesByGraphIds() throws Exception { + public void testGetVerticesByGraphIds() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("g[(a),(b)],h[(a),(c)]", config); @@ -177,7 +177,7 @@ public void testGetVerticesByGraphIds() throws Exception { } @Test - public void testGetVerticesByGraphVariables() throws Exception { + public void testGetVerticesByGraphVariables() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("g[(a),(b)],h[(a),(c)]", config); @@ -210,7 +210,7 @@ public void testGetVerticesByGraphVariables() throws Exception { } @Test - public void testGetEdges() throws Exception { + public void testGetEdges() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("[()-->()]", config); @@ -224,7 +224,7 @@ public void testGetEdges() throws Exception { } @Test - public void testGetEdgesByVariables() throws Exception { + public void testGetEdgesByVariables() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("[()-[e]->()<-[f]-()]", config); @@ -243,7 +243,7 @@ public void testGetEdgesByVariables() throws Exception { } @Test - public void testGetEdgesByGraphIds() throws Exception { + public void testGetEdgesByGraphIds() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("g[()-[a]->()<-[b]-()],h[()-[c]->()-[d]->()]", config); @@ -282,7 +282,7 @@ public void testGetEdgesByGraphIds() throws Exception { } @Test - public void testGetEdgesByGraphVariables() throws Exception { + public void testGetEdgesByGraphVariables() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("g[()-[a]->()<-[b]-()],h[()-[c]->()-[d]->()]", config); @@ -318,7 +318,7 @@ public void testGetEdgesByGraphVariables() throws Exception { } @Test - public void testGetGraphHeadCache() throws Exception { + public void testGetGraphHeadCache() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("g[()],h[()],[()]", config); @@ -337,7 +337,7 @@ public void testGetGraphHeadCache() throws Exception { } @Test - public void testGetVertexCache() throws Exception { + public void testGetVertexCache() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("(a),(b),()", config); @@ -355,7 +355,7 @@ public void testGetVertexCache() throws Exception { } @Test - public void testGetEdgeCache() throws Exception { + public void testGetEdgeCache() { AsciiGraphLoader asciiGraphLoader = AsciiGraphLoader.fromString("()-[e]->()<-[f]-()-->()", config); diff --git a/gradoop-data-integration/pom.xml b/gradoop-data-integration/pom.xml index b4b026989c39..2e08284b9b70 100644 --- a/gradoop-data-integration/pom.xml +++ b/gradoop-data-integration/pom.xml @@ -5,7 +5,7 @@ org.gradoop gradoop-parent - 0.4.4 + 0.4.5 gradoop-data-integration diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/importer/impl/csv/MinimalCSVImporter.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/importer/impl/csv/MinimalCSVImporter.java index 18208e3b8c92..ad09a3517472 100644 --- a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/importer/impl/csv/MinimalCSVImporter.java +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/importer/impl/csv/MinimalCSVImporter.java @@ -16,16 +16,19 @@ package org.gradoop.dataintegration.importer.impl.csv; import org.apache.flink.api.java.DataSet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.gradoop.common.model.impl.pojo.Vertex; -import org.gradoop.dataintegration.importer.impl.csv.functions.PropertiesToVertex; import org.gradoop.dataintegration.importer.impl.csv.functions.CsvRowToProperties; +import org.gradoop.dataintegration.importer.impl.csv.functions.PropertiesToVertex; import org.gradoop.flink.io.api.DataSource; import org.gradoop.flink.model.impl.epgm.GraphCollection; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.util.GradoopFlinkConfig; import java.io.BufferedReader; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.util.Arrays; @@ -172,16 +175,18 @@ private DataSet readCSVFile(List propertyNames, boolean checkReo * @throws IOException if an error occurred while open the stream */ private List readHeaderRow() throws IOException { - try (final BufferedReader reader = - new BufferedReader(new InputStreamReader(new FileInputStream(path), charset))) { - String headerLine = reader.readLine(); + Path filePath = new Path(path); + try (FileSystem fs = FileSystem.get(filePath.toUri(), new Configuration())) { + FSDataInputStream inputStream = fs.open(filePath); + BufferedReader lineReader = new BufferedReader(new InputStreamReader(inputStream, charset)); + String headerLine = lineReader.readLine(); + lineReader.close(); if (headerLine == null || headerLine.isEmpty()) { throw new IOException("The csv file '" + path + "' does not contain any rows."); } - return Arrays.asList(headerLine.split(tokenSeparator)); - } catch (IOException ex) { - throw new IOException("Error while opening a stream to '" + path + "'.", ex); + } catch (IOException ioe) { + throw new IOException("Error while opening a stream to '" + path + "'.", ioe); } } } diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/importer/impl/json/functions/MinimalJsonToVertex.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/importer/impl/json/functions/MinimalJsonToVertex.java index 4b75fdc036e0..2d89452b0b78 100644 --- a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/importer/impl/json/functions/MinimalJsonToVertex.java +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/importer/impl/json/functions/MinimalJsonToVertex.java @@ -69,6 +69,7 @@ public MinimalJsonToVertex(EPGMVertexFactory vertexFactory) { * * @param jsonString The String representation of a JSON object. * @return A new vertex from the JSON object. + * @throws Exception on failure */ @Override public Vertex map(String jsonString) throws Exception { diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/ConnectNeighbors.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/ConnectNeighbors.java new file mode 100644 index 000000000000..dc65b9a0fe09 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/ConnectNeighbors.java @@ -0,0 +1,101 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation; + +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.dataintegration.transformation.functions.CreateCartesianNeighborhoodEdges; +import org.gradoop.dataintegration.transformation.impl.Neighborhood; +import org.gradoop.dataintegration.transformation.impl.NeighborhoodVertex; +import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.functions.epgm.LabelIsIn; + +import java.util.List; +import java.util.Objects; + +/** + * This graph transformation adds new edges to the graph. Those edges are created if a vertex of a + * user-defined label has two neighbors of another user-defined label. A bidirectional (two edges + * in gradoop) edge is then created between those two neighbors. + */ +public class ConnectNeighbors implements UnaryGraphToGraphOperator { + + /** + * The label of the vertices the neighborhood is connected. + */ + private final String sourceVertexLabel; + + /** + * The edge direction to consider. + */ + private final Neighborhood.EdgeDirection edgeDirection; + + /** + * The label of the neighboring vertices that should be connected. + */ + private final String neighborhoodVertexLabel; + + /** + * The label of the created edge between the neighbors. + */ + private final String newEdgeLabel; + + /** + * The constructor to connect the neighbors of vertices with a certain label. + * + * @param sourceVertexLabel The label of the vertices the neighborhood is connected. + * @param edgeDirection The edge direction to consider. + * @param neighborhoodVertexLabel The label of the neighboring vertices that should be connected. + * @param newEdgeLabel The label of the created edge between the neighbors. + */ + public ConnectNeighbors(String sourceVertexLabel, Neighborhood.EdgeDirection edgeDirection, + String neighborhoodVertexLabel, String newEdgeLabel) { + this.sourceVertexLabel = Objects.requireNonNull(sourceVertexLabel); + this.edgeDirection = Objects.requireNonNull(edgeDirection); + this.neighborhoodVertexLabel = Objects.requireNonNull(neighborhoodVertexLabel); + this.newEdgeLabel = Objects.requireNonNull(newEdgeLabel); + } + + @Override + public LogicalGraph execute(LogicalGraph graph) { + + // determine the vertices the neighborhood should be calculated for + DataSet verticesByLabel = graph.getVerticesByLabel(sourceVertexLabel); + + // prepare the graph + LogicalGraph reducedGraph = graph + .vertexInducedSubgraph(new LabelIsIn<>(sourceVertexLabel, neighborhoodVertexLabel)); + + // determine the neighborhood by edge direction + DataSet>> neighborhood = + Neighborhood.getPerVertex(reducedGraph, verticesByLabel, edgeDirection); + + // calculate the new edges and add them to the original graph + DataSet newEdges = neighborhood.flatMap( + new CreateCartesianNeighborhoodEdges<>(graph.getConfig().getEdgeFactory(), newEdgeLabel)); + + return graph.getConfig().getLogicalGraphFactory() + .fromDataSets(graph.getVertices(), graph.getEdges().union(newEdges)); + } + + @Override + public String getName() { + return ConnectNeighbors.class.getName(); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/EdgeToVertex.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/EdgeToVertex.java new file mode 100644 index 000000000000..af2fe48af703 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/EdgeToVertex.java @@ -0,0 +1,106 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation; + +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.tuple.Tuple3; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.dataintegration.transformation.functions.CreateEdgesFromTriple; +import org.gradoop.dataintegration.transformation.functions.CreateVertexFromEdges; +import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.functions.tuple.Value0Of3; + +import java.util.Objects; + +/** + * For edges of a specific label this graph transformation creates a new vertex containing the + * properties of the edge and two new edges respecting the direction of the original edge. + * The newly created edges and vertex labels are user-defined. + *

+ * The original edges are still part of the resulting graph. + * Use a {@link org.apache.flink.api.common.functions.FilterFunction} on the original label to + * remove them. + */ +public class EdgeToVertex implements UnaryGraphToGraphOperator { + + /** + * The label of the edges use for the transformation. + */ + private final String edgeLabel; + + /** + * The label of the newly created vertex. + */ + private final String newVertexLabel; + + /** + * The label of the newly created edge which points to the newly created vertex. + */ + private final String edgeLabelSourceToNew; + + /** + * The label of the newly created edge which starts at the newly created vertex. + */ + private final String edgeLabelNewToTarget; + + /** + * The constructor for the structural transformation. + * + * @param edgeLabel The label of the edges use for the transformation. + * (No edges will be transformed if this parameter is {@code null}). + * @param newVertexLabel The label of the newly created vertex. + * @param edgeLabelSourceToNew The label of the newly created edge which points to the newly + * created vertex. + * @param edgeLabelNewToTarget The label of the newly created edge which starts at the newly + * created vertex. + */ + public EdgeToVertex(String edgeLabel, String newVertexLabel, String edgeLabelSourceToNew, + String edgeLabelNewToTarget) { + this.edgeLabel = edgeLabel; + this.newVertexLabel = Objects.requireNonNull(newVertexLabel); + this.edgeLabelSourceToNew = Objects.requireNonNull(edgeLabelSourceToNew); + this.edgeLabelNewToTarget = Objects.requireNonNull(edgeLabelNewToTarget); + } + + @Override + public LogicalGraph execute(LogicalGraph graph) { + DataSet relevantEdges = graph.getEdgesByLabel(edgeLabel); + + // create new vertices + DataSet> newVerticesAndOriginIds = relevantEdges + .map(new CreateVertexFromEdges<>(newVertexLabel, graph.getFactory().getVertexFactory())); + + DataSet newVertices = newVerticesAndOriginIds + .map(new Value0Of3<>()) + .union(graph.getVertices()); + + // create edges to the newly created vertex + DataSet newEdges = newVerticesAndOriginIds + .flatMap(new CreateEdgesFromTriple<>(graph.getFactory().getEdgeFactory(), + edgeLabelSourceToNew, edgeLabelNewToTarget)) + .union(graph.getEdges()); + + return graph.getFactory().fromDataSets(newVertices, newEdges); + } + + @Override + public String getName() { + return EdgeToVertex.class.getName(); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/PropagatePropertyToNeighbor.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/PropagatePropertyToNeighbor.java new file mode 100644 index 000000000000..52afdb88c6da --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/PropagatePropertyToNeighbor.java @@ -0,0 +1,137 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation; + +import org.apache.flink.api.java.DataSet; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.dataintegration.transformation.functions.AccumulatePropagatedValues; +import org.gradoop.dataintegration.transformation.functions.BuildIdPropertyValuePairs; +import org.gradoop.dataintegration.transformation.functions.BuildTargetVertexIdPropertyValuePairs; +import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.functions.epgm.Id; +import org.gradoop.flink.model.impl.functions.epgm.LabelIsIn; +import org.gradoop.flink.model.impl.functions.epgm.SourceId; + +import java.util.Objects; +import java.util.Set; + +/** + * A property of a vertex is propagated to its neighbors and aggregated in a Property List. + */ +public class PropagatePropertyToNeighbor implements UnaryGraphToGraphOperator { + + /** + * The label of the vertex the property to propagate is part of. + */ + private final String vertexLabel; + + /** + * The property key of the property to propagate. + */ + private final String propertyKey; + + /** + * The property key where the PropertyValue list should be stored at the target vertices. + */ + private final String targetVertexPropertyKey; + + /** + * Only edges with the inserted labels are used. If all labels are sufficient use {@code null}. + */ + private final Set propagatingEdgeLabels; + + /** + * Only vertices with the inserted labels will store the propagated values. + * If all vertices should do it use {@code null}. + */ + private final Set targetVertexLabels; + + /** + * The constructor for the propagate property transformation. Additionally it is possible to + * define which edge labels can be used for propagation and / or which vertices could be target + * of the Properties. + *

+ * Using this constructor, properties will be propagated along all edges and to all + * target vertices. {@link #PropagatePropertyToNeighbor(String, String, String, Set, Set)} + * can be used when properties should only be propagated along certain edges (selected by their + * label) and / or to certain vertices (selected by their label). Using this constructor is + * equivalent to {@code PropagatePropertyToNeighbor(vertexLabel, propertyKey, + * targetVertexPropertyKey, null, null)}. + * + * @param vertexLabel The label of the vertex the property to propagate is part of. + * @param propertyKey The property key of the property to propagate. + * @param targetVertexPropertyKey The property key where the PropertyValue list should be stored + * at the target vertices. + */ + public PropagatePropertyToNeighbor(String vertexLabel, String propertyKey, + String targetVertexPropertyKey) { + this(vertexLabel, propertyKey, targetVertexPropertyKey, null, null); + } + + /** + * The constructor for the propagate property transformation. Additionally it is possible to + * define which edge labels can be used for propagation and / or which vertices could be target + * of the Properties. + * + * @param vertexLabel The label of the vertex the property to propagate is part of. + * @param propertyKey The property key of the property to propagate. + * @param targetVertexPropertyKey The property key where the PropertyValue list should be stored + * at the target vertices. + * @param propagatingEdges Only edges with the inserted labels are used. If all labels + * are sufficient use {@code null}. + * @param targetVertexLabels Only vertices with the inserted labels will store the + * propagated values. If all vertices should, use {@code null}. + */ + public PropagatePropertyToNeighbor(String vertexLabel, String propertyKey, + String targetVertexPropertyKey, Set propagatingEdges, + Set targetVertexLabels) { + this.vertexLabel = Objects.requireNonNull(vertexLabel); + this.propertyKey = Objects.requireNonNull(propertyKey); + this.targetVertexPropertyKey = Objects.requireNonNull(targetVertexPropertyKey); + this.propagatingEdgeLabels = propagatingEdges; + this.targetVertexLabels = targetVertexLabels; + } + + @Override + public LogicalGraph execute(LogicalGraph graph) { + // prepare the edge set, EdgeFilter if propagating edges are given + DataSet propagateAlong = graph.getEdges(); + if (propagatingEdgeLabels != null) { + propagateAlong = propagateAlong.filter(new LabelIsIn<>(propagatingEdgeLabels)); + } + + DataSet newVertices = graph.getVertices() + // Extract properties to propagate + .flatMap(new BuildIdPropertyValuePairs<>(vertexLabel, propertyKey)) + // Propagate along edges. + .join(propagateAlong) + .where(0).equalTo(new SourceId<>()) + .with(new BuildTargetVertexIdPropertyValuePairs<>()) + // Update target vertices. + .coGroup(graph.getVertices()) + .where(0).equalTo(new Id<>()) + .with(new AccumulatePropagatedValues<>(targetVertexPropertyKey, targetVertexLabels)); + + return graph.getFactory().fromDataSets(newVertices, graph.getEdges()); + } + + @Override + public String getName() { + return PropagatePropertyToNeighbor.class.getName(); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/VertexToEdge.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/VertexToEdge.java new file mode 100644 index 000000000000..f10db3f0cf59 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/VertexToEdge.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation; + +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.dataintegration.transformation.functions.EdgesFromLocalTransitiveClosure; +import org.gradoop.dataintegration.transformation.impl.Neighborhood; +import org.gradoop.dataintegration.transformation.impl.NeighborhoodVertex; +import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.neighborhood.keyselector.IdInTuple; + +import java.util.List; +import java.util.Objects; + +/** + * For a given vertex label this graph transformation takes all neighbors per vertex with this + * label and calculates the transitive closure for this subgraph. An edge is created between all + * vertex pairs that fulfill the transitive closure requirement. + * Furthermore each of those created edges contains the labels of the edges to create the + * transitive closure and the former properties of the vertex. + *

+ * Each edge that has to be created results from a path in the graph of the following form:
+ * {@code (i)-[i_j]->(j)-[j_k]->(k)}
+ * The newly created edge goes from: {@code (i)-[e_ik]->(k)}
+ * The edge {@code [e_ik]} has a user-defined label and besides the original vertex properties + * three additional properties: + *

    + *
  • {@code originalVertexLabel}
  • + *
  • {@code firstEdgeLabel = labelOf(i_j)}
  • + *
  • {@code secondEdgeLabel = labelOf(j_k)}
  • + *
+ */ +public class VertexToEdge implements UnaryGraphToGraphOperator { + + /** + * The vertex label of {@code j}. + */ + private final String centralVertexLabel; + + /** + * The edge label for new edges. + */ + private final String newEdgeLabel; + + /** + * The constructor of the operator to transform vertices into edges. + * + * @param centralVertexLabel The vertex label of {@code j}. + * @param newEdgeLabel The edge label for new edges. + */ + public VertexToEdge(String centralVertexLabel, String newEdgeLabel) { + this.centralVertexLabel = Objects.requireNonNull(centralVertexLabel); + this.newEdgeLabel = Objects.requireNonNull(newEdgeLabel); + } + + @Override + public LogicalGraph execute(LogicalGraph graph) { + DataSet>> incomingNeighborhood = Neighborhood + .getPerVertex(graph, graph.getVerticesByLabel(centralVertexLabel), + Neighborhood.EdgeDirection.INCOMING); + + DataSet>> outgoingNeighborhood = Neighborhood + .getPerVertex(graph, graph.getVerticesByLabel(centralVertexLabel), + Neighborhood.EdgeDirection.OUTGOING); + + DataSet newEdges = incomingNeighborhood + .coGroup(outgoingNeighborhood) + .where(new IdInTuple<>(0)) + .equalTo(new IdInTuple<>(0)) + .with(new EdgesFromLocalTransitiveClosure<>(newEdgeLabel, + graph.getFactory().getEdgeFactory())); + + return graph.getFactory().fromDataSets(graph.getVertices(), graph.getEdges().union(newEdges)); + } + + @Override + public String getName() { + return VertexToEdge.class.getName(); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/AccumulatePropagatedValues.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/AccumulatePropagatedValues.java new file mode 100644 index 000000000000..19d5cb56c850 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/AccumulatePropagatedValues.java @@ -0,0 +1,91 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.common.functions.CoGroupFunction; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.api.entities.EPGMVertex; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * This {@link CoGroupFunction} accumulates all properties that might be send to a vertex and + * stores them in a {@link PropertyValue} list. + * + * @param The vertex type. + */ +public class AccumulatePropagatedValues + implements CoGroupFunction, V, V> { + + /** + * The property key where the PropertyValue list should be stored at the target vertices. + */ + private final String targetVertexPropertyKey; + + /** + * Labels of vertices where the propagated property should be set. + */ + private final Set targetVertexLabels; + + /** + * The constructor of the co group function for accumulation of collected property values. + * + * @param targetVertexPropertyKey The property key where the PropertyValue list should be + * stored at the target vertices. + * @param targetVertexLabels The set of labels of elements where the property should be + * set. (Use {@code null} for all vertices.) + */ + public AccumulatePropagatedValues(String targetVertexPropertyKey, + Set targetVertexLabels) { + this.targetVertexPropertyKey = Objects.requireNonNull(targetVertexPropertyKey); + this.targetVertexLabels = targetVertexLabels; + } + + @Override + public void coGroup(Iterable> propertyValues, + Iterable elements, Collector out) { + // should only contain one vertex, based on the uniqueness of gradoop ids + Iterator iterator = elements.iterator(); + if (!iterator.hasNext()) { + return; + } + V targetVertex = iterator.next(); + // If the vertex is not whitelisted by the targetVertexLabels list, + // forward it without modification. + if (targetVertexLabels != null && !targetVertexLabels.contains(targetVertex.getLabel())) { + out.collect(targetVertex); + return; + } + + // collect values of neighbors + List values = new ArrayList<>(); + propertyValues.forEach(t -> values.add(t.f1)); + + // Add to vertex if and only if at least one property was propagated. + if (!values.isEmpty()) { + targetVertex.setProperty(targetVertexPropertyKey, PropertyValue.create(values)); + } + + out.collect(targetVertex); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/BuildIdPropertyValuePairs.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/BuildIdPropertyValuePairs.java new file mode 100644 index 000000000000..b9488063e53f --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/BuildIdPropertyValuePairs.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.api.entities.EPGMElement; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.util.Objects; + +/** + * A simple {@link FlatMapFunction} that prepares element data for further processing. + * Since not all elements necessarily have the property a flat map is used. + * + * @param The element type. + */ +public class BuildIdPropertyValuePairs + implements FlatMapFunction> { + + /** + * The label of elements from which to propagate the property. + */ + private final String label; + + /** + * The property key of the property to propagate. + */ + private final String propertyKey; + + /** + * Reduce object instantiations. + */ + private final Tuple2 reuse; + + /** + * The constructor of the {@link FlatMapFunction} to create {@link GradoopId} / + * {@link PropertyValue} pairs. + * + * @param label The label of the elements to propagate from. + * @param propertyKey The property key of the property to propagate. + */ + public BuildIdPropertyValuePairs(String label, String propertyKey) { + this.label = Objects.requireNonNull(label); + this.propertyKey = Objects.requireNonNull(propertyKey); + this.reuse = new Tuple2<>(); + } + + @Override + public void flatMap(E element, Collector> out) { + if (!label.equals(element.getLabel()) || !element.hasProperty(propertyKey)) { + return; + } + reuse.f0 = element.getId(); + reuse.f1 = element.getPropertyValue(propertyKey); + out.collect(reuse); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/BuildTargetVertexIdPropertyValuePairs.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/BuildTargetVertexIdPropertyValuePairs.java new file mode 100644 index 000000000000..eaf7dc833449 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/BuildTargetVertexIdPropertyValuePairs.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.api.entities.EPGMEdge; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.properties.PropertyValue; + +/** + * The {@link JoinFunction} builds new {@link GradoopId} / {@link PropertyValue} pairs. + * This function is used to propagate a property along an edge. + * + * @param The edge type. + */ +public class BuildTargetVertexIdPropertyValuePairs + implements JoinFunction, E, Tuple2> { + + @Override + public Tuple2 join(Tuple2 t, E e) { + t.f0 = e.getTargetId(); + return t; + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateCartesianNeighborhoodEdges.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateCartesianNeighborhoodEdges.java new file mode 100644 index 000000000000..5ea9f20003ac --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateCartesianNeighborhoodEdges.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.api.entities.EPGMEdge; +import org.gradoop.common.model.api.entities.EPGMEdgeFactory; +import org.gradoop.common.model.api.entities.EPGMVertex; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.dataintegration.transformation.impl.NeighborhoodVertex; + +import java.util.List; +import java.util.Objects; + +/** + * This {@link FlatMapFunction} creates all edges between neighbor vertices. + * + * @param The vertex type. + * @param The edge type. + * @see org.gradoop.dataintegration.transformation.ConnectNeighbors + */ +@FunctionAnnotation.ReadFields({"f1"}) +public class CreateCartesianNeighborhoodEdges + implements FlatMapFunction>, E>, ResultTypeQueryable { + + /** + * The type of the edges created by the factory. + */ + private final Class edgeType; + + /** + * Reduce object instantiations. + */ + private E reuseEdge; + + /** + * The constructor to calculate the edges in the neighborhood. + * + * @param factory The factory the edges are created with. + * @param newEdgeLabel The label of the created edge between the neighbors. + */ + public CreateCartesianNeighborhoodEdges(EPGMEdgeFactory factory, String newEdgeLabel) { + this.edgeType = Objects.requireNonNull(factory).getType(); + this.reuseEdge = factory.createEdge(Objects.requireNonNull(newEdgeLabel), + GradoopId.NULL_VALUE, GradoopId.NULL_VALUE); + } + + @Override + public void flatMap(Tuple2> value, Collector out) { + final List neighbors = value.f1; + + // To "simulate" bidirectional edges we have to create an edge for each direction. + for (NeighborhoodVertex source : neighbors) { + // The source id is the same for the inner loop, we can keep it. + reuseEdge.setSourceId(source.getNeighborId()); + for (NeighborhoodVertex target : neighbors) { + if (source == target) { + continue; + } + reuseEdge.setId(GradoopId.get()); + reuseEdge.setTargetId(target.getNeighborId()); + out.collect(reuseEdge); + } + } + } + + @Override + public TypeInformation getProducedType() { + return TypeInformation.of(edgeType); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateEdgesFromTriple.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateEdgesFromTriple.java new file mode 100644 index 000000000000..edd56074b81a --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateEdgesFromTriple.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.api.entities.EPGMEdge; +import org.gradoop.common.model.api.entities.EPGMEdgeFactory; +import org.gradoop.common.model.api.entities.EPGMVertex; +import org.gradoop.common.model.impl.id.GradoopId; + +import java.util.Objects; + +/** + * A {@link FlatMapFunction} to create two new edges per inserted edge. + * Source to new vertex, new vertex to target. + * + * @param The vertex type. + * @param The edge type. + */ +public class CreateEdgesFromTriple + implements FlatMapFunction, E>, ResultTypeQueryable { + + /** + * The edge type created by the factory. + */ + private final Class edgeType; + + /** + * The label of the newly created edge which points to the newly created vertex. + */ + private final String edgeLabelSourceToNew; + + /** + * The label of the newly created edge which starts at the newly created vertex. + */ + private final String edgeLabelNewToTarget; + + /** + * Reduce object instantiations. + */ + private E reuse; + + /** + * The constructor to create the new edges based on the given triple. + * + * @param factory The Factory which creates the new edges. + * @param edgeLabelSourceToNew The label of the newly created edge which points to the newly + * created vertex. + * @param edgeLabelNewToTarget The label of the newly created edge which starts at the newly + * created vertex. + */ + public CreateEdgesFromTriple(EPGMEdgeFactory factory, String edgeLabelSourceToNew, + String edgeLabelNewToTarget) { + this.edgeType = Objects.requireNonNull(factory).getType(); + this.edgeLabelSourceToNew = Objects.requireNonNull(edgeLabelSourceToNew); + this.edgeLabelNewToTarget = Objects.requireNonNull(edgeLabelNewToTarget); + this.reuse = factory.createEdge(edgeLabelSourceToNew, GradoopId.NULL_VALUE, + GradoopId.NULL_VALUE); + } + + @Override + public void flatMap(Tuple3 triple, Collector out) { + reuse.setId(GradoopId.get()); + reuse.setLabel(edgeLabelSourceToNew); + reuse.setSourceId(triple.f1); + reuse.setTargetId(triple.f0.getId()); + out.collect(reuse); + + reuse.setId(GradoopId.get()); + reuse.setLabel(edgeLabelNewToTarget); + reuse.setSourceId(triple.f0.getId()); + reuse.setTargetId(triple.f2); + out.collect(reuse); + } + + @Override + public TypeInformation getProducedType() { + return TypeInformation.of(edgeType); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateNeighborList.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateNeighborList.java new file mode 100644 index 000000000000..75efe83ca643 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateNeighborList.java @@ -0,0 +1,94 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.common.functions.CoGroupFunction; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.dataintegration.transformation.impl.Neighborhood; +import org.gradoop.dataintegration.transformation.impl.NeighborhoodVertex; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * This CoGroup operation creates a list of neighbors for each vertex. + */ +public class CreateNeighborList + implements CoGroupFunction>> { + + /** + * The edge direction to consider. + */ + private final Neighborhood.EdgeDirection edgeDirection; + + /** + * Reduce object instantiations. + */ + private final Tuple2> reuse; + + /** + * The constructor for the creation of neighbor lists. + * + * @param edgeDirection The edge direction to consider. + */ + public CreateNeighborList(Neighborhood.EdgeDirection edgeDirection) { + this.edgeDirection = Objects.requireNonNull(edgeDirection); + reuse = new Tuple2<>(); + } + + @Override + public void coGroup(Iterable edges, Iterable vertex, + Collector>> out) { + // should only contain one or no vertex + Iterator vertexIterator = vertex.iterator(); + if (vertexIterator.hasNext()) { + Vertex v = vertexIterator.next(); + + List neighbors = new ArrayList<>(); + for (Edge e : edges) { + neighbors.add(new NeighborhoodVertex(getNeighborId(v.getId(), e), e.getLabel())); + } + + reuse.f0 = v; + reuse.f1 = neighbors; + out.collect(reuse); + } + } + + /** + * Based on the considered edge direction the neighbor id is returned. + * + * @param vertexId The vertex id the neighbor id is searched for. + * @param edge The edge the neighbor id is taken from. + * @return The GradoopId of the neighbor vertex. + */ + private GradoopId getNeighborId(GradoopId vertexId, Edge edge) { + switch (edgeDirection) { + case INCOMING: + return edge.getSourceId(); + case OUTGOING: + return edge.getTargetId(); + default: + return vertexId.equals(edge.getSourceId()) ? edge.getTargetId() : edge.getSourceId(); + } + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateVertexFromEdges.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateVertexFromEdges.java new file mode 100644 index 000000000000..fd0c4b8066c9 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/CreateVertexFromEdges.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.gradoop.common.model.api.entities.EPGMEdge; +import org.gradoop.common.model.api.entities.EPGMVertex; +import org.gradoop.common.model.api.entities.EPGMVertexFactory; +import org.gradoop.common.model.impl.id.GradoopId; + +import java.util.Objects; + +/** + * A {@link MapFunction} that creates a new vertex based on the given edge. Furthermore it + * returns the source and target id of the edge for later use. + * + * @param The vertex type. + * @param The edge type. + */ +public class CreateVertexFromEdges + implements MapFunction>, + ResultTypeQueryable> { + + /** + * The factory vertices are created with. + */ + private final EPGMVertexFactory factory; + + /** + * Reduce object instantiations. + */ + private final Tuple3 reuse; + + /** + * The constructor of the MapFunction. + * + * @param newVertexLabel The label of the newly created vertex. + * @param factory The factory for creating new vertices. + */ + public CreateVertexFromEdges(String newVertexLabel, EPGMVertexFactory factory) { + this.factory = Objects.requireNonNull(factory); + this.reuse = new Tuple3<>(factory.createVertex(newVertexLabel), null, null); + } + + @Override + public Tuple3 map(E e) { + reuse.f0.setId(GradoopId.get()); + reuse.f0.setProperties(e.getProperties()); + reuse.f1 = e.getSourceId(); + reuse.f2 = e.getTargetId(); + return reuse; + } + + @Override + public TypeInformation> getProducedType() { + final TypeInformation idType = TypeInformation.of(GradoopId.class); + return new TupleTypeInfo<>(TypeInformation.of(factory.getType()), idType, idType); + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/EdgesFromLocalTransitiveClosure.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/EdgesFromLocalTransitiveClosure.java new file mode 100644 index 000000000000..f7efebbe9813 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/EdgesFromLocalTransitiveClosure.java @@ -0,0 +1,123 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.common.functions.CoGroupFunction; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.api.entities.EPGMEdge; +import org.gradoop.common.model.api.entities.EPGMEdgeFactory; +import org.gradoop.common.model.api.entities.EPGMVertex; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.EdgeFactory; +import org.gradoop.dataintegration.transformation.impl.NeighborhoodVertex; + +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * This function supports the calculation of the transitive closure in the direct neighborhood of a + * vertex. For each transitive connection an edge is created containing the properties of the + * central vertex and the labels of the first and second edge that need to be traversed for the + * transitive connection. + * + * @param The vertex type. + * @param The edge type. + */ +public class EdgesFromLocalTransitiveClosure + implements CoGroupFunction>, + Tuple2>, E>, ResultTypeQueryable { + + /** + * The property key used to store the original vertex label on the new edge. + */ + public static final String ORIGINAL_VERTEX_LABEL = "originalVertexLabel"; + + /** + * The property key used to store the first label of the combined edges on the new edge. + */ + public static final String FIRST_EDGE_LABEL = "firstEdgeLabel"; + + /** + * The property key used to store the second label of the combined edges on the new edge. + */ + public static final String SECOND_EDGE_LABEL = "secondEdgeLabel"; + + /** + * The type of the edges created by the factory.. + */ + private final Class edgeType; + + /** + * Reduce object instantiations. + */ + private final E reuse; + + /** + * The constructor of the CoGroup function to created new edges based on transitivity. + * + * @param newEdgeLabel The edge label of the newly created edge. + * @param factory The {@link EdgeFactory} new edges are created with. + */ + public EdgesFromLocalTransitiveClosure(String newEdgeLabel, EPGMEdgeFactory factory) { + this.edgeType = Objects.requireNonNull(factory).getType(); + this.reuse = factory.createEdge(Objects.requireNonNull(newEdgeLabel), GradoopId.NULL_VALUE, + GradoopId.NULL_VALUE); + } + + @Override + public void coGroup(Iterable>> incoming, + Iterable>> outgoing, + Collector edges) { + + Iterator>> incIt = incoming.iterator(); + Iterator>> outIt = outgoing.iterator(); + + if (incIt.hasNext() && outIt.hasNext()) { + // each of the incoming and outgoing sets should be represented only once. + Tuple2> first = incIt.next(); + V centralVertex = first.f0; + List in = first.f1; + List out = outIt.next().f1; + if (in.isEmpty()) { + return; + } + reuse.setProperties(centralVertex.getProperties()); + reuse.setProperty(ORIGINAL_VERTEX_LABEL, centralVertex.getLabel()); + + for (NeighborhoodVertex source : in) { + for (NeighborhoodVertex target : out) { + reuse.setId(GradoopId.get()); + reuse.setSourceId(source.getNeighborId()); + reuse.setTargetId(target.getNeighborId()); + + reuse.setProperty(FIRST_EDGE_LABEL, source.getConnectingEdgeLabel()); + reuse.setProperty(SECOND_EDGE_LABEL, target.getConnectingEdgeLabel()); + + edges.collect(reuse); + } + } + } + } + + @Override + public TypeInformation getProducedType() { + return TypeInformation.of(edgeType); + } +} diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/package-info.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/package-info.java similarity index 86% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/package-info.java rename to gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/package-info.java index 5ddad1bbd29b..bd8cd0dc8ed0 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/package-info.java +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/functions/package-info.java @@ -14,6 +14,6 @@ * limitations under the License. */ /** - * Contains sampling statistics runner + * Functions used by transformations. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.dataintegration.transformation.functions; diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/ExtractPropertyFromVertex.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/ExtractPropertyFromVertex.java index e096e377e3ab..e322db307081 100644 --- a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/ExtractPropertyFromVertex.java +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/ExtractPropertyFromVertex.java @@ -169,9 +169,4 @@ public LogicalGraph execute(LogicalGraph logicalGraph) { .getLogicalGraphFactory() .fromDataSets(logicalGraph.getGraphHead(), vertices, edges); } - - @Override - public String getName() { - return ExtractPropertyFromVertex.class.getName(); - } } diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/Neighborhood.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/Neighborhood.java new file mode 100644 index 000000000000..2d6ce3cbbd92 --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/Neighborhood.java @@ -0,0 +1,101 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.impl; + +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.dataintegration.transformation.functions.CreateNeighborList; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.functions.epgm.Id; +import org.gradoop.flink.model.impl.functions.epgm.SourceId; +import org.gradoop.flink.model.impl.functions.epgm.TargetId; + +import java.util.List; +import java.util.Objects; + +/** + * This class contains everything related to the neighborhood of vertices. + * A vertex pojo, edge direction and a method to calculate the neighborhood of all vertices. + */ +public class Neighborhood { + + /** + * This methods returns a {@link DataSet} containing a tuple where the first part is a vertex + * and the second one a list of all neighbors of this vertex. The vertices are derived from the + * centralVertices DataSet while the List elements are taken from the graph. + * + * @param graph A Graph the operation is executed on. + * @param centralVertices The vertices the neighborhood should be calculated for. + * @param edgeDirection The relevant direction for neighbors. + * @return A Dataset of tuples containing vertices and their neighborhood. + * @throws NullPointerException if any of the parameters is null. + */ + public static DataSet>> getPerVertex(LogicalGraph graph, + DataSet centralVertices, EdgeDirection edgeDirection) { + Objects.requireNonNull(graph); + Objects.requireNonNull(centralVertices); + Objects.requireNonNull(edgeDirection); + DataSet>> incoming = null; + DataSet>> outgoing = null; + + // get incoming + if (edgeDirection.equals(EdgeDirection.INCOMING) || + edgeDirection.equals(EdgeDirection.UNDIRECTED)) { + incoming = graph.getEdges() + .coGroup(centralVertices) + .where(new TargetId<>()) + .equalTo(new Id<>()) + .with(new CreateNeighborList(edgeDirection)); + } + + // get outgoing + if (edgeDirection.equals(EdgeDirection.OUTGOING) || + edgeDirection.equals(EdgeDirection.UNDIRECTED)) { + outgoing = graph.getEdges() + .coGroup(centralVertices) + .where(new SourceId<>()) + .equalTo(new Id<>()) + .with(new CreateNeighborList(edgeDirection)); + } + + if (edgeDirection.equals(EdgeDirection.UNDIRECTED)) { + return incoming.union(outgoing); + } + return edgeDirection.equals(EdgeDirection.INCOMING) ? incoming : outgoing; + } + + /** + * A simple ENUM which contains possible edge directions viewed from the central vertex. + */ + public enum EdgeDirection { + /** + * Can be used for edges starting from the neighbor to the central vertex. + */ + INCOMING, + + /** + * Can be used for edges starting from the central vertex to the neighbor. + */ + OUTGOING, + + /** + * Can be used if the edge direction should be ignored an INCOMING and OUTGOING edges should be + * taken into account for the calculation. + */ + UNDIRECTED + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/NeighborhoodVertex.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/NeighborhoodVertex.java new file mode 100644 index 000000000000..e58ad4c4162b --- /dev/null +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/NeighborhoodVertex.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.impl; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.id.GradoopId; + +/** + * A simple neighbor vertex tuple which contains information about the Id and label. + */ +public class NeighborhoodVertex extends Tuple2 { + + /** + * A constructor for the Pojo that contains information of a neighbor vertex. + * + * @param neighborId The {@link GradoopId} of the neighbor vertex. + * @param connectingEdgeLabel The edge label of the edge which connects the original vertex and + * the neighbor. + */ + public NeighborhoodVertex(GradoopId neighborId, String connectingEdgeLabel) { + this.f0 = neighborId; + this.f1 = connectingEdgeLabel; + } + + /** + * Get the {@link GradoopId} of the neighbor vertex. + * + * @return GradoopId of the Neighbor. + */ + public GradoopId getNeighborId() { + return f0; + } + + /** + * Get the edge label of the edge which connects the original vertex and the neighbor. + * + * @return The edge label of the connecting edge. + */ + public String getConnectingEdgeLabel() { + return f1; + } +} diff --git a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/PropertyTransformation.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/PropertyTransformation.java index 22bbd5fc05c6..10b1e600a59d 100644 --- a/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/PropertyTransformation.java +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/impl/PropertyTransformation.java @@ -140,9 +140,4 @@ public LG execute(LG graph) { .execute(graph); } - @Override - public String getName() { - return PropertyTransformation.class.getName(); - } - } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/package-info.java b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/package-info.java similarity index 82% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/package-info.java rename to gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/package-info.java index 41a7d047fc94..dc1988634c84 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/package-info.java +++ b/gradoop-data-integration/src/main/java/org/gradoop/dataintegration/transformation/package-info.java @@ -14,6 +14,6 @@ * limitations under the License. */ /** - * Contains a collection of evaluation methods for graph sampling. + * Commonly used vertex-, edge- and property-transformations. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.dataintegration.transformation; diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/ConnectNeighborsTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/ConnectNeighborsTest.java new file mode 100644 index 000000000000..794fb9ae2f27 --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/ConnectNeighborsTest.java @@ -0,0 +1,91 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation; + +import org.gradoop.dataintegration.transformation.impl.Neighborhood; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.junit.Test; + +/** + * Tests for the {@link ConnectNeighbors} operator. + */ +public class ConnectNeighborsTest extends GradoopFlinkTestBase { + + /** + * The loader used to get the test graphs. + */ + private FlinkAsciiGraphLoader loader = getLoaderFromString("input[" + + "(i:V)-->(c:Center)-->(o:V)" + + "(i2:V)-->(c)-->(o2:V)" + + "(:other)-->(c)-->(:other)" + + "] expectedIncoming [" + + "(i)-[:neighbor]->(i2)" + + "(i2)-[:neighbor]->(i)" + + "] expectedOutgoing [" + + "(o)-[:neighbor]->(o2)" + + "(o2)-[:neighbor]->(o)" + + "]"); + + /** + * Test using incoming edges. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testIncoming() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input"); + UnaryGraphToGraphOperator operator = + new ConnectNeighbors("Center", Neighborhood.EdgeDirection.INCOMING, "V", "neighbor"); + LogicalGraph expected = loader.getLogicalGraphByVariable("expectedIncoming").combine(input); + + collectAndAssertTrue(expected.equalsByElementData(input.callForGraph(operator))); + } + + /** + * Test using outgoing edges. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testOutgoing() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input"); + UnaryGraphToGraphOperator operator = + new ConnectNeighbors("Center", Neighborhood.EdgeDirection.OUTGOING, "V", "neighbor"); + LogicalGraph expected = loader.getLogicalGraphByVariable("expectedOutgoing").combine(input); + + collectAndAssertTrue(expected.equalsByElementData(input.callForGraph(operator))); + } + + /** + * Test using edges in both directions. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testUndirected() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input"); + UnaryGraphToGraphOperator operator = + new ConnectNeighbors("Center", Neighborhood.EdgeDirection.UNDIRECTED, "V", "neighbor"); + LogicalGraph expected = loader.getLogicalGraphByVariable("expectedOutgoing") + .combine(loader.getLogicalGraphByVariable("expectedIncoming")) + .combine(input); + + collectAndAssertTrue(expected.equalsByElementData(input.callForGraph(operator))); + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/EdgeToVertexTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/EdgeToVertexTest.java new file mode 100644 index 000000000000..cc56bf8933cd --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/EdgeToVertexTest.java @@ -0,0 +1,103 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation; + +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.junit.Test; + +import static org.junit.Assert.fail; + +/** + * Tests for the {@link EdgeToVertex} operator. + */ +public class EdgeToVertexTest extends GradoopFlinkTestBase { + + /** + * The loader with the graphs used in the tests. + */ + private final FlinkAsciiGraphLoader loader = getLoaderFromString("input[" + + "(a:VertexA)-[e:edgeToTransform {testProp: 1, testProp2: \"\"}]->(b:VertexB)" + + "(a)-[e2:edgeToTransform]->(b)" + + "(a)-[en:anotherEdge]->(b)" + + "]" + + "expected [" + + "(a)-[e]->(b)" + + "(a)-[e2]->(b)" + + "(a)-[en]->(b)" + + "(a)-[:fromSource]->(:VertexFromEdge {testProp: 1, testProp2: \"\"})-[:toTarget]->(b)" + + "(a)-[:fromSource]->(:VertexFromEdge)-[:toTarget]->(b)" + + "]"); + + /** + * Test the {@link EdgeToVertex} operator where one new vertex is created. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testWithVertexCreation() throws Exception { + UnaryGraphToGraphOperator operator = new EdgeToVertex("edgeToTransform", "VertexFromEdge", + "fromSource", "toTarget"); + LogicalGraph result = loader.getLogicalGraphByVariable("input").callForGraph(operator); + LogicalGraph expected = loader.getLogicalGraphByVariable("expected"); + + collectAndAssertTrue(result.equalsByElementData(expected)); + } + + /** + * Test the {@link EdgeToVertex} operator with its first parameter being {@code null}. + * This should not change anything. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testWithoutVertexCreation() throws Exception { + UnaryGraphToGraphOperator operator = new EdgeToVertex(null, "newLabel", "fromSource", + "toTarget"); + LogicalGraph result = loader.getLogicalGraphByVariable("input").callForGraph(operator); + LogicalGraph expected = loader.getLogicalGraphByVariable("input"); + + collectAndAssertTrue(result.equalsByElementData(expected)); + } + + /** + * Test if the constructor {@link EdgeToVertex#EdgeToVertex(String, String, String, String)} + * handles null-values as intended. + */ + @Test + public void testForNullParameters() { + final String notNull = ""; + // Should not throw Exception. + new EdgeToVertex(null, notNull, notNull, notNull); + try { + new EdgeToVertex(null, null, notNull, notNull); + fail("Second parameter was null."); + } catch (NullPointerException npe) { + } + try { + new EdgeToVertex(null, notNull, null, notNull); + fail("Third parameter was null."); + } catch (NullPointerException npe) { + } + try { + new EdgeToVertex(null, notNull, notNull, null); + fail("Forth parameter was null."); + } catch (NullPointerException npe) { + } + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/PropagatePropertyToNeighborTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/PropagatePropertyToNeighborTest.java new file mode 100644 index 000000000000..7c4f2ab8f171 --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/PropagatePropertyToNeighborTest.java @@ -0,0 +1,182 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation; + +import org.gradoop.common.model.impl.properties.PropertyValue; +import org.gradoop.dataintegration.transformation.impl.PropertyTransformation; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * Tests for the {@link PropagatePropertyToNeighbor} operator. + */ +public class PropagatePropertyToNeighborTest extends GradoopFlinkTestBase { + + /** + * A comparator ordering property values by type first. + * This will effectively be able to compare any property value. + * Whether this order makes any sense is not relevant for this test, it is just used to make sure + * that two list-typed properties have the same order. + */ + private static Comparator byTypeFirst = Comparator + .comparing((PropertyValue pv) -> pv.getType().getSimpleName()) + .thenComparing(Comparator.naturalOrder()); + + /** + * The loader with the graphs used in this test. + */ + private FlinkAsciiGraphLoader loader = getLoaderFromString( + "input1[" + "(s1:Source {p1: 1, p2: 1.1d})-[e1:edge1]->(t:Target {t: 0})" + + "(s2:Source {p1: \"\"})-[e2:edge2]->(t)" + + "(s1)-[e12:edge1]->(t2:Target2 {t: 0})" + + "(s2)-[e22:edge2]->(t2)" + + "] input2 [" + + "(v:Vertex {t: 1})-->(v)" + + "]"); + + /** + * Test the operator propagating a property to two vertices. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testPropagateDirected() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input1"); + UnaryGraphToGraphOperator operator = new PropagatePropertyToNeighbor("Source", "p1", "t"); + // We have to update the vertices manually because the ascii loader does not support lists. + LogicalGraph expected = input.transformVertices((v, c) -> { + if (!v.getLabel().equals("Target") && !v.getLabel().equals("Target2")) { + return v; + } + v.setProperty("t", Arrays.asList(PropertyValue.create(1L), PropertyValue.create(""))); + return v; + }).callForGraph(new PropertyTransformation<>("t", pv -> pv, + PropagatePropertyToNeighborTest::orderListProperty, pv -> pv)); + LogicalGraph result = input.callForGraph(operator) + .callForGraph(new PropertyTransformation<>("t", pv -> pv, + PropagatePropertyToNeighborTest::orderListProperty, pv -> pv)); + collectAndAssertTrue(expected.equalsByElementData(result)); + } + + /** + * Test the operator propagating along only certain edge labels. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testPropagateAlongCertainEdges() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input1"); + UnaryGraphToGraphOperator operator = new PropagatePropertyToNeighbor("Source", "p1", "t", + Collections.singleton("edge1"), null); + LogicalGraph expected = input.transformVertices((v, c) -> { + if (!v.getLabel().equals("Target") && !v.getLabel().equals("Target2")) { + return v; + } + v.setProperty("t", Collections.singletonList(PropertyValue.create(1L))); + return v; + }); + LogicalGraph result = input.callForGraph(operator); + collectAndAssertTrue(expected.equalsByElementData(result)); + } + + /** + * Test the operator propagating only to vertices with a certain label. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testPropagateToCertainVertices() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input1"); + UnaryGraphToGraphOperator operator = new PropagatePropertyToNeighbor("Source", "p1", "t", + null, Collections.singleton("Target")); + LogicalGraph expected = input.transformVertices((v, c) -> { + if (!v.getLabel().equals("Target")) { + return v; + } + v.setProperty("t", Arrays.asList(PropertyValue.create(1L), PropertyValue.create(""))); + return v; + }).callForGraph(new PropertyTransformation<>("t", pv -> pv, + PropagatePropertyToNeighborTest::orderListProperty, pv -> pv)); + LogicalGraph result = input.callForGraph(operator) + .callForGraph(new PropertyTransformation<>("t", pv -> pv, + PropagatePropertyToNeighborTest::orderListProperty, pv -> pv)); + collectAndAssertTrue(expected.equalsByElementData(result)); + } + + /** + * Test the operator propagating only to vertices with a certain label and only along edges of + * a certain label. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testPropagateToCertainVerticesAlongCertainEdges() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input1"); + UnaryGraphToGraphOperator operator = new PropagatePropertyToNeighbor("Source", "p1", "t", + Collections.singleton("edge1"), Collections.singleton("Target")); + LogicalGraph expected = input.transformVertices((v, c) -> { + if (!v.getLabel().equals("Target")) { + return v; + } + v.setProperty("t", Collections.singletonList(PropertyValue.create(1L))); + return v; + }); + LogicalGraph result = input.callForGraph(operator); + collectAndAssertTrue(expected.equalsByElementData(result)); + } + + /** + * Test if the operator works correctly for loops. + * This will also check if using the same property key for reading and writing values works. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testPropagateInLoops() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input2"); + UnaryGraphToGraphOperator operator = new PropagatePropertyToNeighbor("Vertex", "t", "t"); + LogicalGraph expected = input.transformVertices((v, c) -> { + v.setProperty("t", Collections.singletonList(PropertyValue.create(1L))); + return v; + }); + LogicalGraph result = input.callForGraph(operator); + collectAndAssertTrue(expected.equalsByElementData(result)); + } + + /** + * Order a list-type property. This is used to make sure that two lists are equal except + * for the order of their elements. + * + * @param list The property. + * @return The ordered property. + */ + private static PropertyValue orderListProperty(PropertyValue list) { + if (!list.isList()) { + return list; + } + List theList = list.getList(); + theList.sort(byTypeFirst); + return PropertyValue.create(theList); + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/VertexToEdgeTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/VertexToEdgeTest.java new file mode 100644 index 000000000000..607bfe18a745 --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/VertexToEdgeTest.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation; + +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.junit.Test; + +/** + * This class contains tests for the {@link VertexToEdge} transformation operator. + */ +public class VertexToEdgeTest extends GradoopFlinkTestBase { + + /** + * Test the {@link VertexToEdge} transformation where one one edge is added. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testWithEdgeCreation() throws Exception { + FlinkAsciiGraphLoader loader = getLoaderFromString("input[" + + "(v0:Blue {a : 3})" + + "(v1:Green {a : 2})" + + "(v2:Blue {a : 4})" + + "(v0)-[{b : 2}]->(v1)" + + "(v1)-[{b : 4}]->(v2)" + + "]" + + "expected[" + + "(v00:Blue {a : 3})" + + "(v01:Green {a : 2})" + + "(v02:Blue {a : 4})" + + "(v00)-[{b : 2}]->(v01)" + + "(v01)-[{b : 4}]->(v02)" + + "(v00)-[:foo {a : 2, originalVertexLabel: \"Green\"," + + "firstEdgeLabel: \"\", secondEdgeLabel: \"\"}]->(v02)" + + "]"); + LogicalGraph input = loader.getLogicalGraphByVariable("input"); + LogicalGraph expected = loader.getLogicalGraphByVariable("expected"); + + VertexToEdge transformation = new VertexToEdge("Green", "foo"); + LogicalGraph transformed = input.callForGraph(transformation); + + collectAndAssertTrue(transformed.equalsByElementData(expected)); + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/AccumulatePropagatedValuesTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/AccumulatePropagatedValuesTest.java new file mode 100644 index 000000000000..c04954bcf783 --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/AccumulatePropagatedValuesTest.java @@ -0,0 +1,66 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.pojo.VertexFactory; +import org.gradoop.common.model.impl.properties.PropertyValue; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.functions.epgm.Id; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; + +/** + * Test for the {@link AccumulatePropagatedValues} function used in + * {@link org.gradoop.dataintegration.transformation.PropagatePropertyToNeighbor}. + */ +public class AccumulatePropagatedValuesTest extends GradoopFlinkTestBase { + + /** + * Test the coGroup function using some values. + * + * @throws Exception on failure + */ + @Test + public void testCoGroup() throws Exception { + VertexFactory vertexFactory = getConfig().getVertexFactory(); + Vertex v1 = vertexFactory.createVertex("a"); + Tuple2 property1 = Tuple2.of(v1.getId(), PropertyValue.create(1L)); + Vertex v2 = vertexFactory.createVertex("a"); + Vertex v3 = vertexFactory.createVertex("b"); + Tuple2 property2 = Tuple2.of(v3.getId(), PropertyValue.create(1L)); + List input = Arrays.asList(v1, v2, v3); + List result = getExecutionEnvironment().fromElements(property1, property2) + .coGroup(getExecutionEnvironment().fromCollection(input)) + .where(0).equalTo(new Id<>()) + .with(new AccumulatePropagatedValues<>("k", Collections.singleton("a"))) + .collect(); + v1.setProperty("k", PropertyValue.create(Collections.singletonList(PropertyValue.create(1L)))); + List expected = Arrays.asList(v1, v2, v3); + Comparator comparator = Comparator.comparing(Vertex::getId); + expected.sort(comparator); + result.sort(comparator); + assertArrayEquals(expected.toArray(), result.toArray()); + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/BuildIdPropertyValuePairsTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/BuildIdPropertyValuePairsTest.java new file mode 100644 index 000000000000..a257707ffc79 --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/BuildIdPropertyValuePairsTest.java @@ -0,0 +1,58 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.pojo.VertexFactory; +import org.gradoop.common.model.impl.properties.PropertyValue; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Test for the {@link BuildIdPropertyValuePairs} function used by + * {@link org.gradoop.dataintegration.transformation.PropagatePropertyToNeighbor}. + */ +public class BuildIdPropertyValuePairsTest extends GradoopFlinkTestBase { + + /** + * Test if the function selects the correct labels and properties. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testFunction() throws Exception { + VertexFactory vertexFactory = getConfig().getVertexFactory(); + Vertex v1 = vertexFactory.createVertex("a"); + Vertex v2 = vertexFactory.createVertex("a"); + v2.setProperty("k1", 1L); + v2.setProperty("k2", 1L); + Vertex v3 = vertexFactory.createVertex("b"); + v3.setProperty("k1", 1L); + v3.setProperty("k2", 1L); + Vertex v4 = vertexFactory.createVertex(); + List> result = getExecutionEnvironment() + .fromElements(v1, v2, v3, v4).flatMap(new BuildIdPropertyValuePairs<>("a", "k1")) + .collect(); + assertEquals(1, result.size()); + assertEquals(Tuple2.of(v2.getId(), PropertyValue.create(1L)), result.get(0)); + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/BuildTargetVertexIdPropertyValuePairsTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/BuildTargetVertexIdPropertyValuePairsTest.java new file mode 100644 index 000000000000..471cfd4e78fc --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/BuildTargetVertexIdPropertyValuePairsTest.java @@ -0,0 +1,65 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.properties.PropertyValue; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.functions.epgm.SourceId; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Test for the {@link BuildTargetVertexIdPropertyValuePairs} join function used by + * {@link org.gradoop.dataintegration.transformation.PropagatePropertyToNeighbor}. + */ +public class BuildTargetVertexIdPropertyValuePairsTest extends GradoopFlinkTestBase { + + /** + * Test the join function by using some values. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testJoinFunction() throws Exception { + Comparator> comparator = Comparator.comparing(t -> t.f0); + GradoopId source1 = GradoopId.get(); + GradoopId target1 = GradoopId.get(); + GradoopId source2 = GradoopId.get(); + GradoopId target2 = GradoopId.get(); + Edge edge1 = getConfig().getEdgeFactory().createEdge(source1, target1); + Edge edge2 = getConfig().getEdgeFactory().createEdge(source2, target2); + Tuple2 tuple1 = new Tuple2<>(source1, PropertyValue.create(1L)); + Tuple2 tuple2 = new Tuple2<>(source2, PropertyValue.create(2L)); + List> result = getExecutionEnvironment() + .fromElements(tuple1, tuple2) + .join(getExecutionEnvironment().fromElements(edge1, edge2)).where(0) + .equalTo(new SourceId<>()).with(new BuildTargetVertexIdPropertyValuePairs<>()).collect(); + result.sort(comparator); + Tuple2 expected1 = new Tuple2<>(target1, PropertyValue.create(1L)); + Tuple2 expected2 = new Tuple2<>(target2, PropertyValue.create(2L)); + List> expected = Arrays.asList(expected1, expected2); + expected.sort(comparator); + assertArrayEquals(expected.toArray(), result.toArray()); + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateCartesianNeighborhoodEdgesTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateCartesianNeighborhoodEdgesTest.java new file mode 100644 index 000000000000..694f2b010af4 --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateCartesianNeighborhoodEdgesTest.java @@ -0,0 +1,111 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.pojo.VertexFactory; +import org.gradoop.dataintegration.transformation.impl.NeighborhoodVertex; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +/** + * Test for the {@link CreateCartesianNeighborhoodEdges} function used by + * {@link org.gradoop.dataintegration.transformation.ConnectNeighbors}. + */ +public class CreateCartesianNeighborhoodEdgesTest extends GradoopFlinkTestBase { + + /** + * The label used for the newly created edges. + */ + private final String edgeLabel = "test"; + + /** + * The function to test. + */ + private CreateCartesianNeighborhoodEdges toTest; + + /** + * The factory used to create new vertices. + */ + private VertexFactory vertexFactory; + + /** + * Set this test up, creating the function to test. + */ + @Before + public void setUp() { + vertexFactory = getConfig().getVertexFactory(); + toTest = new CreateCartesianNeighborhoodEdges<>(getConfig().getEdgeFactory(), edgeLabel); + } + + /** + * Test the function using an empty neighborhood. Should produce no edges. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testWithEmptyNeighborhood() throws Exception { + Vertex someVertex = vertexFactory.createVertex(); + Tuple2> inputEmpty = new Tuple2<>(someVertex, + Collections.emptyList()); + List result = getExecutionEnvironment().fromElements(inputEmpty) + .flatMap(toTest).collect(); + assertEquals(0, result.size()); + } + + /** + * Test the function using a non-empty neighborhood. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testWithNonEmptyNeighborhood() throws Exception { + Vertex someVertex = vertexFactory.createVertex(); + final int count = 10; + List ids = Stream.generate(GradoopId::get).limit(count).collect(Collectors.toList()); + // Create some dummy neighborhood vertex pojos. + List vertexPojos = ids.stream() + .map(id -> new NeighborhoodVertex(id, "")) + .collect(Collectors.toList()); + Tuple2> inputNonEmpty = new Tuple2<>(someVertex, + vertexPojos); + List result = getExecutionEnvironment().fromElements(inputNonEmpty) + .flatMap(toTest).collect(); + // Connect each neighbor with another neighbor, except for itself. + assertEquals(count * (count - 1), result.size()); + // The result should not contain loops + for (Edge edge : result) { + assertNotEquals(edge.getSourceId(), edge.getTargetId()); + } + // or duplicate edges. + long disctinctCount = result.stream() + .map(e -> new Tuple2<>(e.getSourceId(), e.getTargetId())) + .distinct().count(); + assertEquals((long) result.size(), disctinctCount); + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateEdgesFromTripleTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateEdgesFromTripleTest.java new file mode 100644 index 000000000000..bc6c3c14aadf --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateEdgesFromTripleTest.java @@ -0,0 +1,88 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple3; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.pojo.VertexFactory; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Test for the {@link CreateEdgesFromTriple} function used by + * {@link org.gradoop.dataintegration.transformation.EdgeToVertex}. + */ +public class CreateEdgesFromTripleTest extends GradoopFlinkTestBase { + + /** + * Test the function by applying it to some tuples. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testFunction() throws Exception { + CreateEdgesFromTriple function = new CreateEdgesFromTriple<>( + getConfig().getEdgeFactory(), "source", "target"); + VertexFactory vertexFactory = getConfig().getVertexFactory(); + Vertex testVertex1 = vertexFactory.createVertex(); + Vertex testVertex2 = vertexFactory.createVertex(); + GradoopId source1 = GradoopId.get(); + GradoopId source2 = GradoopId.get(); + GradoopId target1 = GradoopId.get(); + GradoopId target2 = GradoopId.get(); + Tuple3 tuple1 = new Tuple3<>(testVertex1, source1, target1); + Tuple3 tuple2 = new Tuple3<>(testVertex2, source2, target2); + List result = getExecutionEnvironment().fromElements(tuple1, tuple2).flatMap(function) + .collect(); + // Check if the correct number of edges were created and if they are distinct. + assertEquals(4, result.size()); + // By id. + assertEquals(4, result.stream().map(Element::getId).count()); + // By source and target id. + assertEquals(4, + result.stream().map(e -> Tuple2.of(e.getSourceId(), e.getTargetId())).distinct().count()); + // Finally check the data of the edges. + for (Edge resultEdge : result) { + if (resultEdge.getLabel().equals("source")) { + if (resultEdge.getSourceId().equals(source1)) { + assertEquals(testVertex1.getId(), resultEdge.getTargetId()); + } else if (resultEdge.getSourceId().equals(source2)) { + assertEquals(testVertex2.getId(), resultEdge.getTargetId()); + } else { + fail("Edge with invalid source ID created."); + } + } else if (resultEdge.getLabel().equals("target")) { + if (resultEdge.getSourceId().equals(testVertex1.getId())) { + assertEquals(target1, resultEdge.getTargetId()); + } else if (resultEdge.getSourceId().equals(testVertex2.getId())) { + assertEquals(target2, resultEdge.getTargetId()); + } else { + fail("Edge with invalid source ID created."); + } + } else { + fail("Edge with invalid label created."); + } + } + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateVertexFromEdgesTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateVertexFromEdgesTest.java new file mode 100644 index 000000000000..05967220605d --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/CreateVertexFromEdgesTest.java @@ -0,0 +1,65 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.java.tuple.Tuple3; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.EdgeFactory; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Test for the {@link CreateVertexFromEdges} function used by + * {@link org.gradoop.dataintegration.transformation.EdgeToVertex}. + */ +public class CreateVertexFromEdgesTest extends GradoopFlinkTestBase { + /** + * Test the function by applying it to some vertices. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testFunction() throws Exception { + CreateVertexFromEdges function = new CreateVertexFromEdges<>("test", + getConfig().getVertexFactory()); + GradoopId dummy = GradoopId.get(); + EdgeFactory edgeFactory = getConfig().getEdgeFactory(); + // Create some test edges, with some having no properties or label. + Edge withoutProperties = edgeFactory.createEdge(dummy, dummy); + Edge withProperties = edgeFactory.createEdge("TestEdge2", dummy, dummy); + withProperties.setProperty("TestProperty", 1L); + List edges = Arrays.asList(withoutProperties, withProperties); + List> result = getExecutionEnvironment() + .fromCollection(edges).map(function).collect(); + // There should be a new vertex for each edge. + assertEquals(edges.size(), result.size()); + // Every ID should be assigned only once. + long idCount = result.stream().map(t -> t.f0.getId()).distinct().count(); + assertEquals("Vertex IDs are not unique.", edges.size(), idCount); + for (Tuple3 resultTuple : result) { + if (resultTuple.f0.getPropertyCount() > 0) { + assertEquals(withProperties.getProperties(), resultTuple.f0.getProperties()); + } + } + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/NeighborhoodTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/NeighborhoodTest.java new file mode 100644 index 000000000000..3b4d9a726a34 --- /dev/null +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/functions/NeighborhoodTest.java @@ -0,0 +1,119 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.dataintegration.transformation.functions; + +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.dataintegration.transformation.impl.Neighborhood; +import org.gradoop.dataintegration.transformation.impl.NeighborhoodVertex; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +/** + * Tests for {@link Neighborhood}. + */ +@RunWith(Parameterized.class) +public class NeighborhoodTest extends GradoopFlinkTestBase { + + /** + * Loader to get the test graphs from. + */ + private final FlinkAsciiGraphLoader loader = getLoaderFromString("input[" + + "(i1:I1)-->(center:Center)-->(o1:O1)" + + "(center2:Center2)" + + "]" + + "incoming1 [(i1)]" + + "outgoing1 [(o1)]" + + "undirected1 [(i1)(o1)]" + + "incoming2[] outgoing2[] undirected2[]"); + + /** + * The variable name of the center vertex. + */ + @Parameterized.Parameter(0) + public String centerVertex; + + /** + * The variable name of the neighborhood graph. + */ + @Parameterized.Parameter(1) + public String neighborhood; + + /** + * The direction of the neighborhood. + */ + @Parameterized.Parameter(2) + public Neighborhood.EdgeDirection direction; + + /** + * Parameters for this test. + * + * @return the parameters for this test. + */ + @Parameterized.Parameters(name = "neighborhood of {0} ({2})") + public static Iterable parameters() { + return Arrays.asList( + new Object[]{"center", "incoming1", Neighborhood.EdgeDirection.INCOMING}, + new Object[]{"center", "outgoing1", Neighborhood.EdgeDirection.OUTGOING}, + new Object[]{"center", "undirected1", Neighborhood.EdgeDirection.UNDIRECTED}, + new Object[]{"center2", "incoming2", Neighborhood.EdgeDirection.INCOMING}, + new Object[]{"center2", "outgoing2", Neighborhood.EdgeDirection.OUTGOING}, + new Object[]{"center2", "undirected2", Neighborhood.EdgeDirection.UNDIRECTED}, + new Object[]{"nocenter", "incoming2", Neighborhood.EdgeDirection.INCOMING}, + new Object[]{"nocenter", "outgoing2", Neighborhood.EdgeDirection.OUTGOING}, + new Object[]{"nocenter", "undirected2", Neighborhood.EdgeDirection.UNDIRECTED}); + } + + /** + * Test {@link Neighborhood} with different parameters. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testNeighborhood() throws Exception { + LogicalGraph input = loader.getLogicalGraphByVariable("input"); + Vertex center = loader.getVertexByVariable(centerVertex); + Collection neighborhoodVertices = loader.getVerticesByGraphVariables(neighborhood); + DataSet centers = center == null ? getEmptyDataSet(new Vertex()) : + getExecutionEnvironment().fromElements(center); + List>> neighborhoods = + Neighborhood.getPerVertex(input, centers, direction).collect(); + // Make sure we only have 1 center vertex. + long centerCount = neighborhoods.stream().map(h -> h.f0.getId()).distinct().count(); + assertEquals(center != null ? 1 : 0, centerCount); + + // Get all neighbor ids of that vertex. + Object[] neighbors = neighborhoods.stream().flatMap(h -> h.f1.stream()) + .map(NeighborhoodVertex::getNeighborId) + .sorted() + .toArray(); + Object[] neighborIdsExpected = neighborhoodVertices.stream().map(Vertex::getId).sorted() + .toArray(); + assertArrayEquals(neighborIdsExpected, neighbors); + } +} diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/impl/ExtractPropertyFromVertexTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/impl/ExtractPropertyFromVertexTest.java index 06fc227a68cd..25706592e53d 100644 --- a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/impl/ExtractPropertyFromVertexTest.java +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/impl/ExtractPropertyFromVertexTest.java @@ -26,7 +26,7 @@ import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.epgm.ByLabel; -import org.gradoop.flink.model.impl.operators.subgraph.functions.LabelIsIn; +import org.gradoop.flink.model.impl.functions.epgm.LabelIsIn; import org.junit.Assert; import org.junit.Test; @@ -155,8 +155,9 @@ private void testForEdgeDirection(LogicalGraph graph, EdgeDirection direction) t } else if (direction.equals(EdgeDirection.BIDIRECTIONAL)) { boolean cityContainment = cities.contains(sourceName) || cities.contains(targetName); boolean personContainment = persons.contains(sourceName) || persons.contains(targetName); - Assert.assertTrue("vertex name 1: " + sourceName + " | vertex name 2: " + targetName + - " | edge direction: " + direction.name(), + + Assert.assertTrue("vertex name 1: " + sourceName + " | vertex name 2: " + + targetName + " | edge direction: " + direction.name(), cityContainment && personContainment); } } @@ -203,12 +204,15 @@ public void nonDeduplicationTest() throws Exception { public void listPropertyTest() throws Exception { VertexFactory vf = getConfig().getVertexFactory(); Vertex v1 = vf.createVertex("foo"); - v1.setProperty("a", PropertyValue.create(Arrays.asList(PropertyValue.create("m"), PropertyValue.create("n")))); + v1.setProperty("a", PropertyValue.create(Arrays.asList(PropertyValue.create("m"), + PropertyValue.create("n")))); Vertex v2 = vf.createVertex("foo"); - v2.setProperty("a", PropertyValue.create(Arrays.asList(PropertyValue.create("x"), PropertyValue.create("y"), PropertyValue.create("z")))); + v2.setProperty("a", PropertyValue.create(Arrays.asList(PropertyValue.create("x"), + PropertyValue.create("y"), PropertyValue.create("z")))); - LogicalGraph input = getConfig().getLogicalGraphFactory().fromCollections(Arrays.asList(v1, v2), Collections.emptyList()); + LogicalGraph input = getConfig().getLogicalGraphFactory().fromCollections( + Arrays.asList(v1, v2), Collections.emptyList()); ExtractPropertyFromVertex ext = new ExtractPropertyFromVertex("foo", "a", "A", "key"); LogicalGraph output = input.callForGraph(ext); diff --git a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/impl/InvertEdgesTest.java b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/impl/InvertEdgesTest.java index c5614aa911e4..1ae0c7fef364 100644 --- a/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/impl/InvertEdgesTest.java +++ b/gradoop-data-integration/src/test/java/org/gradoop/dataintegration/transformation/impl/InvertEdgesTest.java @@ -79,18 +79,17 @@ public void testInvert() throws Exception { long oldEdgeCount = invertedEdgeGraph.getEdges().filter(new ByLabel<>(toInvertLabel)).count(); Assert.assertEquals(oldEdgeCount, 0); // no edges with the old label should exist - //------------------- - // We now have to check whether all of these hasInterest edges are inverted. - // - // (eve)-[:hasInterest]->(databases) - // (alice)-[:hasInterest]->(databases) - // (frank)-[:hasInterest]->(hadoop) - // (dave)-[:hasInterest]->(hadoop) - //------------------- - long invertedEdgeCount = invertedEdgeGraph.getEdges().filter(new ByLabel<>(invertedLabel)).count(); Assert.assertEquals(edgesToChange, invertedEdgeCount); + /* + * We now have to check whether all of these hasInterest edges are inverted. + * (eve)-[:hasInterest]->(databases) + * (alice)-[:hasInterest]->(databases) + * (frank)-[:hasInterest]->(hadoop) + * (dave)-[:hasInterest]->(hadoop) + */ + List vertices = new ArrayList<>(); invertedEdgeGraph.getVertices() .filter(new Or<>(new ByLabel<>("Person"), new ByLabel<>("Tag"))) diff --git a/gradoop-examples/pom.xml b/gradoop-examples/pom.xml index 152b3d310180..7852ea26831c 100644 --- a/gradoop-examples/pom.xml +++ b/gradoop-examples/pom.xml @@ -1,11 +1,12 @@ - + 4.0.0 org.gradoop gradoop-parent - 0.4.4 + 0.4.5 gradoop-examples @@ -112,6 +113,7 @@ org.apache.hbase:hbase-common org.apache.flink:flink-gelly_2.11 + org.neo4j:* org.gradoop:* com.github.s1ck:gdl org.codehaus.jettison:jettison @@ -119,6 +121,17 @@ org.antlr:antlr4-runtime me.lemire.integercompression:* commons-cli:* + + org.opencypher:* + org.apache.flink:flink-gelly_2.11 + org.apache.flink:flink-table_2.11 + org.parboiled:parboiled-scala_2.11 + com.lihaoyi:ujson_2.11 + com.lihaoyi:upickle_2.11 + org.typelevel:cats-kernel_2.11 + org.typelevel:cats-core_2.11 + org.atnos:eff_2.11 + org.parboiled:* @@ -175,6 +188,5 @@ commons-cli commons-cli - \ No newline at end of file diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CAPFBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CAPFBenchmark.java new file mode 100644 index 000000000000..3e1590597e88 --- /dev/null +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CAPFBenchmark.java @@ -0,0 +1,246 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.benchmark.cypher; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.io.FileUtils; +import org.apache.flink.api.common.ProgramDescription; +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.api.java.io.LocalCollectionOutputFormat; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.examples.AbstractRunner; +import org.gradoop.flink.model.impl.epgm.GraphCollection; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.CAPFQueryResult; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * A dedicated program for parametrized graph cypher benchmark. + */ +public class CAPFBenchmark extends AbstractRunner implements ProgramDescription { + /** + * Option for used first name in query. + */ + private static final String OPTION_FIRST_NAME = "n"; + /** + * Option to declare path to indexed input graph + */ + private static final String OPTION_INPUT_PATH = "i"; + /** + * Option to declare output path to statistics csv file + */ + private static final String OPTION_CSV_PATH = "o"; + /** + * Option to use a custom query string + */ + private static final String OPTION_CUSTOM_QUERY = "cq"; + /** + * Option to use a predefined query string + */ + private static final String OPTION_PREDEFINED_QUERY = "pq"; + /** + * Used input path + */ + private static String INPUT_PATH; + /** + * Used output path for csv statistics + */ + private static String CSV_PATH; + /** + * Used query + */ + private static String QUERY; + /** + * Used first name for query (q1,q2,q3) + */ + private static String FIRST_NAME; + + static { + OPTIONS.addOption(OPTION_INPUT_PATH, "input", true, + "Input path to indexed source files."); + OPTIONS.addOption(OPTION_CSV_PATH, "csv", true, + "Output path to csv statistics output"); + OPTIONS.addOption(OPTION_CUSTOM_QUERY, "customQuery", true, + "Custom query specified by user"); + OPTIONS.addOption(OPTION_PREDEFINED_QUERY, "predefinedQuery", true, + "Predefined cypher query"); + OPTIONS.addOption(OPTION_FIRST_NAME, "firstName", true, + "Used first Name in Cypher Query"); + } + + /** + * Main program to run the benchmark. Arguments are the available options. + * + * @param args program arguments + * @throws Exception IO or execution Exception + */ + public static void main(String[] args) throws Exception { + CommandLine cmd = parseArguments(args, CAPFBenchmark.class.getName()); + + if (cmd == null) { + System.exit(1); + } + + // test if minimum arguments are set + performSanityCheck(cmd); + + // read cmd arguments + readCMDArguments(cmd); + + // read graph + LogicalGraph graph = readLogicalGraph(INPUT_PATH); + + // prepare collection + GraphCollection collection; + + System.out.println(QUERY); + // execute cypher with or without statistics + CAPFQueryResult result = graph.cypher(QUERY); + + + if (result.containsGraphs()) { + collection = result.getGraphs(); + List vertices = new ArrayList<>(); + collection.getVertices().output(new LocalCollectionOutputFormat<>(vertices)); + System.out.println(collection.getConfig().getExecutionEnvironment().getExecutionPlan()); +// System.out.println(collection.getGraphHeads().count()); + } else { + result.getTable().printSchema(); + } + + // execute and write job statistics + writeCSV(graph.getConfig().getExecutionEnvironment()); + } + + /** + * Reads the given arguments from command line + * + * @param cmd command line + */ + private static void readCMDArguments(CommandLine cmd) { + INPUT_PATH = cmd.getOptionValue(OPTION_INPUT_PATH); + CSV_PATH = cmd.getOptionValue(OPTION_CSV_PATH); + FIRST_NAME = cmd.getOptionValue(OPTION_FIRST_NAME); + + if (cmd.hasOption(OPTION_CUSTOM_QUERY)) { + StringBuilder queryBuilder = new StringBuilder(); + for (String val : cmd.getOptionValues(OPTION_CUSTOM_QUERY)) { + queryBuilder.append(val).append(" "); + } + QUERY = queryBuilder.toString(); + } else if (cmd.hasOption(OPTION_PREDEFINED_QUERY)) { + QUERY = getPredefinedQuery(cmd.getOptionValue(OPTION_PREDEFINED_QUERY)); + } + } + + /** + * Checks if the minimum of arguments is provided + * + * @param cmd command line + */ + private static void performSanityCheck(CommandLine cmd) { + if (!cmd.hasOption(OPTION_INPUT_PATH)) { + throw new IllegalArgumentException("Define a graph input directory."); + } + if (!cmd.hasOption(OPTION_CSV_PATH)) { + throw new IllegalArgumentException("Path to CSV-File need to be set."); + } + if (!(cmd.hasOption(OPTION_PREDEFINED_QUERY) || cmd.hasOption(OPTION_CUSTOM_QUERY))) { + throw new IllegalArgumentException("Define a query to run."); + } + if (cmd.hasOption(OPTION_PREDEFINED_QUERY) && cmd.hasOption(OPTION_CUSTOM_QUERY)) { + throw new IllegalArgumentException("Specify either a custom or predefined query, not both!"); + } + if (cmd.hasOption(OPTION_PREDEFINED_QUERY)) { + String predefQuery = cmd.getOptionValue(OPTION_PREDEFINED_QUERY); + if (predefQuery.equals("q1") || predefQuery.equals("q2") || predefQuery.equals("q3")) { + if (!cmd.hasOption(OPTION_FIRST_NAME)) { + throw new IllegalArgumentException("Queries 1, 2 and 3 require a first name."); + } + } + } + } + + /** + * Method to create and add lines to a csv-file + * + * @param env given ExecutionEnvironment + * @throws IOException exeption during file writing + */ + private static void writeCSV(ExecutionEnvironment env) throws IOException { + + String head = String.format("%s|%s|%s|%s%n", + "Parallelism", + "dataset", + "query", + "Runtime(s)"); + + String tail = String.format("%s|%s|%s|%s%n", + env.getParallelism(), + INPUT_PATH, + QUERY, + env.getLastJobExecutionResult().getNetRuntime(TimeUnit.SECONDS)); + + File f = new File(CSV_PATH); + if (f.exists() && !f.isDirectory()) { + FileUtils.writeStringToFile(f, tail, true); + } else { + PrintWriter writer = new PrintWriter(CSV_PATH, "UTF-8"); + writer.print(head); + writer.print(tail); + writer.close(); + } + } + + /** + * Returns used query for benchmark + * + * @param query argument input + * @return used query + */ + private static String getPredefinedQuery(String query) { + switch (query) { + case "q1": + return CAPFQueries.q1(FIRST_NAME); + case "q2": + return CAPFQueries.q2(FIRST_NAME); + case "q3": + return CAPFQueries.q3(FIRST_NAME); + case "q4": + return CAPFQueries.q4(); + case "q5": + return CAPFQueries.q5(); + case "q6": + return CAPFQueries.q6(); + default: + throw new IllegalArgumentException("Unsupported query: " + query); + } + } + + /** + * {@inheritDoc} + */ + @Override + public String getDescription() { + return CAPFBenchmark.class.getName(); + } +} diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CAPFQueries.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CAPFQueries.java new file mode 100644 index 000000000000..5a4bfd44b8c1 --- /dev/null +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CAPFQueries.java @@ -0,0 +1,104 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.benchmark.cypher; + +/** + * Used Queries for {@link CypherBenchmark} + */ +class CAPFQueries { + + /** + * Operational Query 1 + * + * @param name used first name in query + * @return query string + */ + static String q1(String name) { + return + "MATCH (p:person)<-[:hasCreator]-(c:comment), " + + "(p)<-[:hasCreator]-(po:post) " + + "WHERE p.firstName = '" + name + "' RETURN *"; + } + + /** + * Operational Query 2 + * + * @param name used first name in query + * @return query string + */ + static String q2(String name) { + return + "MATCH (p:person)<-[:hasCreator]-(c:comment)," + + "(p)<-[:hasCreator]-(po:post)," + + "(c)-[:replyOf]->(po)" + + "WHERE p.firstName = '" + name + "' RETURN *"; + } + + /** + * Operational Query 3 + * + * @param name used first name in query + * @return query string + */ + static String q3(String name) { + return + "MATCH (p1:person )-[:knows]->(p2:person)," + + "(c:comment)-[:hasCreator]->(p2)," + + "(c)-[:replyOf]->(po:post)," + + "(po)-[:hasCreator]->(p1)" + + "WHERE p1.firstName = '" + name + "' RETURN *"; + + } + + /** + * Analytical Query 1 + * + * @return query string + */ + static String q4() { + return + "MATCH (p:person)-[:isLocatedIn]->(c:city)," + + "(p)-[:hasInterest]->(t:tag)," + + "(p)-[:studyAt]->(u:university)," + + "(p)<-[:hasMember]-(f:forum)," + + "(p)<-[:hasModerator]-(f) RETURN *"; + } + + /** + * Analytical Query 2 + * + * @return query string + */ + static String q5() { + return + "MATCH (p1:person)-[:knows]->(p2:person)," + + "(p2)-[:knows]->(p3:person)," + + "(p1)-[:knows]->(p3) RETURN *"; + } + + /** + * Analytical Query 3 + * + * @return query string + */ + static String q6() { + return + "MATCH (p1:person)-[:knows]->(p2:person)," + + "(p1)-[:hasInterest]->(t1:tag)," + + "(p2)-[:hasInterest]->(t1)," + + "(p2)-[:hasInterest]->(t2:tag) RETURN *"; + } +} diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CypherBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CypherBenchmark.java index 6feaf2399ab6..cc91c2f51cdf 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CypherBenchmark.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/cypher/CypherBenchmark.java @@ -235,9 +235,6 @@ private static void writeCSV(ExecutionEnvironment env) throws IOException { } } - /** - * {@inheritDoc} - */ @Override public String getDescription() { return CypherBenchmark.class.getName(); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/grouping/GroupingBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/grouping/GroupingBenchmark.java index 40532d22a36e..d6858774fe1c 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/grouping/GroupingBenchmark.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/grouping/GroupingBenchmark.java @@ -205,7 +205,7 @@ public class GroupingBenchmark extends AbstractRunner * Main program to run the benchmark. Arguments are the available options. * * @param args program arguments - * @throws Exception + * @throws Exception on failure */ @SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { @@ -458,7 +458,7 @@ private static Grouping getOperator(GroupingStrategy strategy, /** * Method to create and add lines to a csv-file - * @throws IOException + * @throws IOException on failure */ private static void writeCSV() throws IOException { @@ -486,9 +486,6 @@ private static void writeCSV() throws IOException { } } - /** - * {@inheritDoc} - */ @Override public String getDescription() { return GroupingBenchmark.class.getName(); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/SetPairTraverserBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/SetPairTraverserBenchmark.java index 9095694f6fb5..f0194c6e222b 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/SetPairTraverserBenchmark.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/SetPairTraverserBenchmark.java @@ -60,6 +60,7 @@ private SetPairTraverserBenchmark(CommandLine cmd) { * -t,--traverser [loop|bulk] * * @param args program arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { CommandLine cmd = parseArguments(args, SetPairTraverserBenchmark.class.getName()); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TransactionalBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TransactionalBenchmark.java index 46bf16c625bc..fe8e943ba66c 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TransactionalBenchmark.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TransactionalBenchmark.java @@ -80,6 +80,7 @@ public class TransactionalBenchmark extends AbstractRunner { * -t,--traverser [loop|bulk] * * @param args program arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { CommandLine cmd = @@ -148,7 +149,7 @@ private static void performSanityCheck(final CommandLine cmd) { private static class SecondFieldTrue implements FilterFunction> { @Override - public boolean filter(Tuple2 tuple2) throws Exception { + public boolean filter(Tuple2 tuple2) { return tuple2.f1; } } diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TraverserBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TraverserBenchmark.java index fb47f322f6f2..3a4ad22de0bf 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TraverserBenchmark.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TraverserBenchmark.java @@ -170,7 +170,7 @@ private String getResultString() { * the results are appended. * * @param csvFile path to csv file - * @throws IOException + * @throws IOException on failure */ private void writeResults(String csvFile) throws IOException { String header = "Input|Parallelism|Strategy|Query|Embeddings|Runtime[ms]"; @@ -221,13 +221,15 @@ TraverserStrategy getTraverserStrategy() { /** * Run the benchmark. + * + * @throws Exception on failure */ abstract void run() throws Exception; /** * Writes the results to file or prints it. * - * @throws IOException + * @throws IOException on failure */ void close() throws IOException { if (csvPath != null) { @@ -308,7 +310,7 @@ public static class GetIdWithCandidates } @Override - public IdWithCandidates map(Tuple1 value) throws Exception { + public IdWithCandidates map(Tuple1 value) { reuseTuple.setId(value.f0); return reuseTuple; } @@ -341,7 +343,7 @@ public static class GetTriplesWithCandidates implements @Override public TripleWithCandidates map( - Tuple3 value) throws Exception { + Tuple3 value) { reuseTuple.setEdgeId(value.f0); reuseTuple.setSourceId(value.f1); reuseTuple.setTargetId(value.f2); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TripleTraverserBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TripleTraverserBenchmark.java index 4c0ac9a06887..d57a534aef0a 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TripleTraverserBenchmark.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/patternmatching/TripleTraverserBenchmark.java @@ -59,6 +59,7 @@ private TripleTraverserBenchmark(CommandLine cmd) { * -t,--traverser [loop|bulk] * * @param args program arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { CommandLine cmd = parseArguments(args, TripleTraverserBenchmark.class.getName()); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/sampling/SamplingBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/sampling/SamplingBenchmark.java index ba908a610a78..9ccbf0192dd4 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/sampling/SamplingBenchmark.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/sampling/SamplingBenchmark.java @@ -55,6 +55,7 @@ public class SamplingBenchmark extends AbstractRunner implements ProgramDescript * 4 ---> RandomVertexEdgeSampling * 5 ---> RandomVertexNeighborhoodSampling * 6 ---> RandomVertexSampling + * 7 ---> RandomWalkSampling */ private static final String OPTION_SELECTED_ALGORITHM = "a"; /** @@ -120,6 +121,7 @@ public class SamplingBenchmark extends AbstractRunner implements ProgramDescript * Other arguments are the available options. * * @param args program arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { CommandLine cmd = parseArguments(args, SamplingBenchmark.class.getName()); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/sampling/SamplingBuilder.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/sampling/SamplingBuilder.java index 0aae240bcdfc..e31d89b4095b 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/sampling/SamplingBuilder.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/sampling/SamplingBuilder.java @@ -22,6 +22,7 @@ import org.gradoop.flink.model.impl.operators.sampling.RandomLimitedDegreeVertexSampling; import org.gradoop.flink.model.impl.operators.sampling.RandomEdgeSampling; import org.gradoop.flink.model.impl.operators.sampling.PageRankSampling; +import org.gradoop.flink.model.impl.operators.sampling.RandomWalkSampling; import org.gradoop.flink.model.impl.operators.sampling.SamplingAlgorithm; /** @@ -47,7 +48,9 @@ private enum Algorithm { /** RandomVertexNeighborhoodSampling enum constant */ RANDOM_VERTEX_NEIGHBORHOOD_SAMPLING(RandomVertexNeighborhoodSampling.class.getSimpleName()), /** RandomVertexSampling enum constant */ - RANDOM_VERTEX_SAMPLING(RandomVertexSampling.class.getSimpleName()); + RANDOM_VERTEX_SAMPLING(RandomVertexSampling.class.getSimpleName()), + /** RandomWalkSampling enum constant */ + RANDOM_WALK_SAMPLING(RandomWalkSampling.class.getSimpleName()); /** Property denoting the simple classname of a sampling algorithm */ private final String name; @@ -160,6 +163,19 @@ static SamplingAlgorithm buildSelectedSamplingAlgorithm(int ordinal, String[] co new String[]{"1", "2"}, constructorParams.length); } + case RANDOM_WALK_SAMPLING: + if (constructorParams.length == 2) { + return new RandomWalkSampling(Float.parseFloat(constructorParams[0]), + Integer.parseInt(constructorParams[1])); + } else if (constructorParams.length == 4) { + return new RandomWalkSampling(Float.parseFloat(constructorParams[0]), + Integer.parseInt(constructorParams[1]), Float.parseFloat(constructorParams[2]), + Integer.parseInt(constructorParams[3])); + } else { + throw createInstantiationException(Algorithm.RANDOM_WALK_SAMPLING.name, + new String[]{"1", "2"}, constructorParams.length); + } + default: throw new IllegalArgumentException( "Something went wrong. Please select an other sampling algorithm."); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/sna/SocialNetworkAnalyticsExample.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/sna/SocialNetworkAnalyticsExample.java index 9d87b86302fb..e29e28b1c8b6 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/sna/SocialNetworkAnalyticsExample.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/sna/SocialNetworkAnalyticsExample.java @@ -33,7 +33,7 @@ import org.gradoop.flink.model.impl.operators.grouping.GroupingStrategy; import org.gradoop.flink.model.impl.operators.matching.common.statistics.GraphStatistics; import org.gradoop.flink.model.impl.operators.matching.common.statistics.GraphStatisticsHDFSReader; -import org.gradoop.flink.model.impl.operators.subgraph.functions.LabelIsIn; +import org.gradoop.flink.model.impl.functions.epgm.LabelIsIn; import org.gradoop.flink.util.GradoopFlinkConfig; import java.io.File; @@ -237,9 +237,6 @@ private static void writeCSV(ExecutionEnvironment env) throws IOException { } } - /** - * {@inheritDoc} - */ @Override public String getDescription() { return SocialNetworkAnalyticsExample.class.getName(); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/sna/functions/CountFilter.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/sna/functions/CountFilter.java index 5a80c407c58a..a442bdcaf566 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/sna/functions/CountFilter.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/sna/functions/CountFilter.java @@ -23,9 +23,7 @@ * Filter all edges below a count of 500 */ public class CountFilter implements FilterFunction { - /** - * {@inheritDoc} - */ + @Override public boolean filter(Edge edge) { PropertyValue value = edge.getPropertyValue("count"); diff --git a/gradoop-examples/src/main/java/org/gradoop/benchmark/subgraph/SubgraphBenchmark.java b/gradoop-examples/src/main/java/org/gradoop/benchmark/subgraph/SubgraphBenchmark.java index 15692f17a809..b693012d4eed 100644 --- a/gradoop-examples/src/main/java/org/gradoop/benchmark/subgraph/SubgraphBenchmark.java +++ b/gradoop-examples/src/main/java/org/gradoop/benchmark/subgraph/SubgraphBenchmark.java @@ -205,9 +205,6 @@ private static void writeCSV(ExecutionEnvironment env) throws IOException { } } - /** - * {@inheritDoc} - */ @Override public String getDescription() { return SubgraphBenchmark.class.getName(); diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/AbstractRunner.java b/gradoop-examples/src/main/java/org/gradoop/examples/AbstractRunner.java index cc34b1d53b3d..caf823459746 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/AbstractRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/AbstractRunner.java @@ -78,6 +78,7 @@ protected static CommandLine parseArguments(String[] args, String className) * * @param directory path to EPGM database * @return EPGM logical graph + * @throws IOException on failure */ protected static LogicalGraph readLogicalGraph(String directory) throws IOException { return readLogicalGraph(directory, DEFAULT_FORMAT); @@ -210,9 +211,8 @@ protected static void convertDotToPNG(String dotFile, String pngFile) throws IOE * @param directory input path * @param format format in which the data is stored (csv, indexed, json) * @return DataSource for EPGM Data - * @throws IOException on failure */ - private static DataSource getDataSource(String directory, String format) throws IOException { + private static DataSource getDataSource(String directory, String format) { directory = appendSeparator(directory); GradoopFlinkConfig config = GradoopFlinkConfig.createConfig(getExecutionEnvironment()); format = format.toLowerCase(); diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/aggregation/AggregationExample.java b/gradoop-examples/src/main/java/org/gradoop/examples/aggregation/AggregationExample.java index 0bf35b8d07b3..de1c1e905e22 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/aggregation/AggregationExample.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/aggregation/AggregationExample.java @@ -69,6 +69,7 @@ public class AggregationExample { * * @see * Gradoop Wiki + * @throws Exception on failure */ public static void main(String[] args) throws Exception { // init execution environment diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/aggregation/functions/AggregateListOfNames.java b/gradoop-examples/src/main/java/org/gradoop/examples/aggregation/functions/AggregateListOfNames.java index 435389b046c7..df0952abfff6 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/aggregation/functions/AggregateListOfNames.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/aggregation/functions/AggregateListOfNames.java @@ -15,7 +15,7 @@ */ package org.gradoop.examples.aggregation.functions; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.api.functions.VertexAggregateFunction; import org.gradoop.flink.model.impl.operators.aggregation.functions.BaseAggregateFunction; @@ -40,7 +40,7 @@ public AggregateListOfNames() { } @Override - public PropertyValue getIncrement(Element vertex) { + public PropertyValue getIncrement(EPGMElement vertex) { return PropertyValue.create(vertex.getPropertyValue(PROPERTY_KEY_NAME).toString()); } diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/biiig/CategoryCharacteristicPatterns.java b/gradoop-examples/src/main/java/org/gradoop/examples/biiig/CategoryCharacteristicPatterns.java index e0ff4d18dfd7..2517e57aa97b 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/biiig/CategoryCharacteristicPatterns.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/biiig/CategoryCharacteristicPatterns.java @@ -18,15 +18,15 @@ import org.apache.commons.io.IOUtils; import org.apache.flink.api.common.ProgramDescription; import org.apache.flink.api.java.ExecutionEnvironment; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.examples.utils.ExampleOutput; import org.gradoop.flink.algorithms.btgs.BusinessTransactionGraphs; import org.gradoop.flink.algorithms.fsm.transactional.CategoryCharacteristicSubgraphs; import org.gradoop.flink.algorithms.fsm.transactional.common.FSMConfig; +import org.gradoop.flink.model.api.functions.VertexAggregateFunction; import org.gradoop.flink.model.impl.epgm.GraphCollection; import org.gradoop.flink.model.impl.epgm.LogicalGraph; -import org.gradoop.flink.model.api.functions.VertexAggregateFunction; import org.gradoop.flink.model.impl.operators.aggregation.ApplyAggregation; import org.gradoop.flink.model.impl.operators.aggregation.functions.bool.Or; import org.gradoop.flink.model.impl.operators.aggregation.functions.count.VertexCount; @@ -54,8 +54,9 @@ public class CategoryCharacteristicPatterns implements ProgramDescription { /** * main method + * * @param args arguments (none required) - * @throws Exception + * @throws Exception on failure */ public static void main(String[] args) throws Exception { @@ -114,8 +115,9 @@ public static void main(String[] args) throws Exception { /** * Returns example integrated instance graph from GDL input. + * * @return integrated instance graph - * @throws IOException + * @throws IOException on failure */ public static LogicalGraph getIntegratedInstanceGraph() throws IOException { @@ -154,7 +156,7 @@ public String getAggregatePropertyKey() { } @Override - public PropertyValue getIncrement(Element vertex) { + public PropertyValue getIncrement(EPGMElement vertex) { boolean isClosedQuotation = vertex.getLabel().equals("Quotation") && !vertex.getPropertyValue("status").toString().equals("open"); @@ -176,7 +178,7 @@ private static class CountSalesOrdersAggregateFunction extends VertexCount { } @Override - public PropertyValue getIncrement(Element vertex) { + public PropertyValue getIncrement(EPGMElement vertex) { return PropertyValue.create(vertex.getLabel().equals("SalesOrder") ? 1 : 0); } } diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/biiig/FrequentLossPatterns.java b/gradoop-examples/src/main/java/org/gradoop/examples/biiig/FrequentLossPatterns.java index 19c3e7054809..f5bb921d9ed2 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/biiig/FrequentLossPatterns.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/biiig/FrequentLossPatterns.java @@ -16,8 +16,8 @@ package org.gradoop.examples.biiig; import org.apache.flink.api.common.ProgramDescription; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.pojo.Edge; -import org.gradoop.common.model.impl.pojo.Element; import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.common.model.impl.pojo.Vertex; import org.gradoop.common.model.impl.properties.PropertyValue; @@ -236,7 +236,7 @@ private static class Result implements Sum, VertexAggregateFunction { @Override - public PropertyValue getIncrement(Element vertex) { + public PropertyValue getIncrement(EPGMElement vertex) { PropertyValue increment; if (vertex.hasProperty(REVENUE_KEY)) { @@ -262,7 +262,7 @@ public String getAggregatePropertyKey() { private static class DetermineMasterDataSurplus implements Sum, VertexAggregateFunction { @Override - public PropertyValue getIncrement(Element vertex) { + public PropertyValue getIncrement(EPGMElement vertex) { return vertex.getLabel().startsWith(MASTER_PREFIX) ? PropertyValue.create(1) : PropertyValue.create(-1); } diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/DIMSpanRunner.java b/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/DIMSpanRunner.java index 5bb624ec1a30..dc8180def6ed 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/DIMSpanRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/DIMSpanRunner.java @@ -67,7 +67,7 @@ public class DIMSpanRunner extends AbstractRunner implements ProgramDescription * Main program to run the example. Arguments are the available options. * * @param args program arguments - * @throws Exception + * @throws Exception on failure */ @SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { @@ -120,9 +120,6 @@ private static void performSanityCheck(final CommandLine cmd) { } } - /** - * {@inheritDoc} - */ @Override public String getDescription() { return DIMSpanRunner.class.getName(); diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/TLFDataCopier.java b/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/TLFDataCopier.java index 78843dcb405a..b581961e89ba 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/TLFDataCopier.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/TLFDataCopier.java @@ -72,7 +72,7 @@ public class TLFDataCopier extends AbstractRunner implements ProgramDescription * Main program to run the example. Arguments are the available options. * * @param args program arguments - * @throws Exception + * @throws Exception on failure */ @SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { @@ -130,9 +130,6 @@ private static void performSanityCheck(final CommandLine cmd) { } } - /** - * {@inheritDoc} - */ @Override public String getDescription() { return TLFDataCopier.class.getName(); diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/data_source/DIMSpanGraphFromText.java b/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/data_source/DIMSpanGraphFromText.java index 784cfa4ac3f7..45493310ad8c 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/data_source/DIMSpanGraphFromText.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/data_source/DIMSpanGraphFromText.java @@ -38,7 +38,7 @@ public class DIMSpanGraphFromText public static final char COLUMN_SEPARATOR = ' '; @Override - public LabeledGraphStringString map(Tuple2 inputTuple) throws Exception { + public LabeledGraphStringString map(Tuple2 inputTuple) { LabeledGraphStringString setPair = LabeledGraphStringString.getEmptyOne(); // create character array diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/data_source/DIMSpanTLFSource.java b/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/data_source/DIMSpanTLFSource.java index f0d93461b588..03053a53c0d9 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/data_source/DIMSpanTLFSource.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/dimspan/data_source/DIMSpanTLFSource.java @@ -62,6 +62,7 @@ public DIMSpanTLFSource(String filePath, GradoopFlinkConfig config) { * Reads the input as dataset of TLFGraphs. * * @return io graphs + * @throws IOException on failure */ public DataSet getGraphs() throws IOException { ExecutionEnvironment env = getConfig().getExecutionEnvironment(); diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/grouping/Composition.java b/gradoop-examples/src/main/java/org/gradoop/examples/grouping/Composition.java index 764ec25c6cde..3b58a30a4636 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/grouping/Composition.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/grouping/Composition.java @@ -27,7 +27,6 @@ import org.gradoop.flink.model.impl.operators.grouping.GroupingStrategy; import org.gradoop.flink.util.GradoopFlinkConfig; -import java.io.IOException; import java.time.LocalDate; import java.util.Arrays; import java.util.Collections; @@ -48,7 +47,7 @@ public class Composition extends AbstractRunner { * args[1] - output path * * @param args arguments - * @throws IOException if something goes wrong + * @throws Exception if something goes wrong */ public static void main(String[] args) throws Exception { String inputPath = args[0]; diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/grouping/GroupingRunner.java b/gradoop-examples/src/main/java/org/gradoop/examples/grouping/GroupingRunner.java index 8fa333e58ef1..a793c0cca219 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/grouping/GroupingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/grouping/GroupingRunner.java @@ -158,9 +158,6 @@ private static void performSanityCheck(final CommandLine cmd) { } } - /** - * {@inheritDoc} - */ @Override public String getDescription() { return GroupingRunner.class.getName(); diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/io/DOTExample.java b/gradoop-examples/src/main/java/org/gradoop/examples/io/DOTExample.java index 246602892d72..e8d1af9e63b4 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/io/DOTExample.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/io/DOTExample.java @@ -40,6 +40,7 @@ public class DOTExample extends AbstractRunner implements ProgramDescription { * args[3]: flag to write graph head information * * @param args program arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { if (args.length != 5) { diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/io/EdgeListExample.java b/gradoop-examples/src/main/java/org/gradoop/examples/io/EdgeListExample.java index 9a1d2e34d4f6..d050f2df5730 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/io/EdgeListExample.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/io/EdgeListExample.java @@ -56,6 +56,7 @@ public class EdgeListExample implements ProgramDescription { * args[1]: token separator (optional, default is single whitespace) * * @param args program arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { if (args.length == 0) { diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/io/JSONExample.java b/gradoop-examples/src/main/java/org/gradoop/examples/io/JSONExample.java index 70c986df89e9..b6a8bdd075ae 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/io/JSONExample.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/io/JSONExample.java @@ -77,6 +77,7 @@ public class JSONExample extends AbstractRunner implements ProgramDescription { * args[1]: path to output graph * * @param args program arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { if (args.length != 2) { diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/io/PokecExample.java b/gradoop-examples/src/main/java/org/gradoop/examples/io/PokecExample.java index 5a6695aba5cf..01caa4bbe778 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/io/PokecExample.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/io/PokecExample.java @@ -138,7 +138,7 @@ public class PokecExample { * args[0]: path to directory that contains pokec files (e.g. hdfs:///pokec/) * args[1]: path to write the output graph to (e.g. hdfs:///output/) * @param args arguments - * @throws Exception + * @throws Exception on failure */ public static void main(String[] args) throws Exception { final String inputDir = args[0]; @@ -171,7 +171,7 @@ public static void main(String[] args) throws Exception { @SuppressWarnings("Duplicates") @Override - public ImportVertex map(String line) throws Exception { + public ImportVertex map(String line) { String[] fields = line.split(splitPattern.pattern()); importVertex.setId(Long.parseLong(fields[0])); // user-id @@ -262,7 +262,7 @@ public ImportVertex map(String line) throws Exception { @Override public ImportEdge map( - Tuple2> value) throws Exception { + Tuple2> value) { importEdge.setId(value.f0); importEdge.setSourceId(value.f1.f0); importEdge.setTargetId(value.f1.f1); diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/patternmatching/PatternMatchingRunner.java b/gradoop-examples/src/main/java/org/gradoop/examples/patternmatching/PatternMatchingRunner.java index d429e1b4f78d..2e6d870ad71a 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/patternmatching/PatternMatchingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/patternmatching/PatternMatchingRunner.java @@ -131,6 +131,7 @@ public class PatternMatchingRunner extends AbstractRunner implements ProgramDesc * -s, --statistics (path to input graph statistics, used for cypher engine)
* * @param args option line + * @throws Exception on failure */ @SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/quickstart/GradoopQuickstart.java b/gradoop-examples/src/main/java/org/gradoop/examples/quickstart/GradoopQuickstart.java new file mode 100644 index 000000000000..4a5f8f836f04 --- /dev/null +++ b/gradoop-examples/src/main/java/org/gradoop/examples/quickstart/GradoopQuickstart.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.examples.quickstart; + +import org.apache.flink.api.java.ExecutionEnvironment; +import org.gradoop.flink.algorithms.gelly.connectedcomponents.WeaklyConnectedComponentsAsCollection; +import org.gradoop.flink.io.api.DataSink; +import org.gradoop.flink.io.impl.dot.DOTDataSink; +import org.gradoop.flink.model.impl.epgm.GraphCollection; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.gradoop.flink.util.GradoopFlinkConfig; + +/** + * Simple Gradoop Example that walks through the process of loading data, doing a simple graph + * transformation and storing the results + * */ +public class GradoopQuickstart { + + /** + * run the example + * + * @param args no args used + * @throws Exception on failure + */ + public static void main(String[] args) throws Exception { + + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + GradoopFlinkConfig cfg = GradoopFlinkConfig.createConfig(env); + + // Create the sample Graphs + String graph = "g1:graph[" + + "(p1:Person {name: \"Bob\", age: 24})-[:friendsWith]->" + + "(p2:Person{name: \"Alice\", age: 30})-[:friendsWith]->(p1)" + + "(p2)-[:friendsWith]->(p3:Person {name: \"Jacob\", age: 27})-[:friendsWith]->(p2) " + + "(p3)-[:friendsWith]->(p4:Person{name: \"Marc\", age: 40})-[:friendsWith]->(p3) " + + "(p4)-[:friendsWith]->(p5:Person{name: \"Sara\", age: 33})-[:friendsWith]->(p4) " + + "(c1:Company {name: \"Acme Corp\"}) " + + "(c2:Company {name: \"Globex Inc.\"}) " + + "(p2)-[:worksAt]->(c1) " + + "(p4)-[:worksAt]->(c1) " + + "(p5)-[:worksAt]->(c1) " + + "(p1)-[:worksAt]->(c2) " + + "(p3)-[:worksAt]->(c2) " + "] " + + "g2:graph[" + + "(p4)-[:friendsWith]->(p6:Person {name: \"Paul\", age: 37})-[:friendsWith]->(p4) " + + "(p6)-[:friendsWith]->(p7:Person {name: \"Mike\", age: 23})-[:friendsWith]->(p6) " + + "(p8:Person {name: \"Jil\", age: 32})-[:friendsWith]->(p7)-[:friendsWith]->(p8) " + + "(p6)-[:worksAt]->(c2) " + + "(p7)-[:worksAt]->(c2) " + + "(p8)-[:worksAt]->(c1) " + "]"; + + //LOAD INPUT GRAPHS (from above) + FlinkAsciiGraphLoader loader = new FlinkAsciiGraphLoader(cfg); + loader.initDatabaseFromString(graph); + + GraphCollection inputGraphs = loader.getGraphCollectionByVariables("g1", "g2"); + DataSink inputGraphSink = new DOTDataSink("out/input.dot", true); + inputGraphs.writeTo(inputGraphSink, true); + + + // OVERLAP + LogicalGraph graph1 = loader.getLogicalGraphByVariable("g1"); + LogicalGraph graph2 = loader.getLogicalGraphByVariable("g2"); + LogicalGraph overlap = graph2.overlap(graph1); + DataSink overlapSink = new DOTDataSink("out/overlap.dot", true); + overlap.writeTo(overlapSink, true); + + LogicalGraph workGraph = graph1.combine(graph2) + .subgraph( + v -> true, + e -> e.getLabel().equals("worksAt")); + + // Weakly Connected Components Algorithm + WeaklyConnectedComponentsAsCollection weaklyConnectedComponents = + new WeaklyConnectedComponentsAsCollection(10); + GraphCollection workspaces = weaklyConnectedComponents.execute(workGraph); + + DataSink workspaceSink = new DOTDataSink("out/workspace.dot", true); + workspaces.writeTo(workspaceSink, true); + + env.execute(); + } + +} diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/quickstart/package-info.java b/gradoop-examples/src/main/java/org/gradoop/examples/quickstart/package-info.java new file mode 100644 index 000000000000..8e5c2c8065bf --- /dev/null +++ b/gradoop-examples/src/main/java/org/gradoop/examples/quickstart/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Simple Gradoop Example that walks through the process of loading data, doing a simple graph + * transformation and storing the results + */ +package org.gradoop.examples.quickstart; diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/rollup/RollUpExample.java b/gradoop-examples/src/main/java/org/gradoop/examples/rollup/RollUpExample.java index b061b3295214..7ca075e033ca 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/rollup/RollUpExample.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/rollup/RollUpExample.java @@ -63,6 +63,7 @@ public class RollUpExample extends AbstractRunner { * * @see * Gradoop Wiki + * @throws Exception on failure */ public static void main(String[] args) throws Exception { // init execution environment diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/sna/SNABenchmark1.java b/gradoop-examples/src/main/java/org/gradoop/examples/sna/SNABenchmark1.java index a31b5a57df0b..7a91e38296cb 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/sna/SNABenchmark1.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/sna/SNABenchmark1.java @@ -49,7 +49,7 @@ public class SNABenchmark1 extends AbstractRunner implements * Needs a (possibly HDFS) output directory to write the resulting graph to. * * @param args args[0] = input dir, args[1] input format, args[2] output dir - * @throws Exception + * @throws Exception on failure */ @SuppressWarnings({ "unchecked", diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/sna/SNABenchmark2.java b/gradoop-examples/src/main/java/org/gradoop/examples/sna/SNABenchmark2.java index 7e420d4a7f99..c3f218ca3dde 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/sna/SNABenchmark2.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/sna/SNABenchmark2.java @@ -82,7 +82,7 @@ public class SNABenchmark2 extends AbstractRunner implements * 10K - 2,450,000 * * @param args args[0]: input dir, args[1]: input format, args[2]: output dir, args[3]: threshold - * @throws Exception + * @throws Exception on failure */ @SuppressWarnings({ "unchecked", @@ -105,7 +105,7 @@ public static void main(String[] args) throws Exception { * in a given format (csv, indexed, json) * * @param args args[0]: input dir, args[1]: input format, args[2]: output dir, args[3]: threshold - * @throws Exception + * @throws Exception on failure */ @SuppressWarnings("unchecked") private static void executeWithExternalData(String[] args) throws Exception { @@ -126,7 +126,7 @@ private static void executeWithExternalData(String[] args) throws Exception { * Runs the benchmark with demo data. * * @param gradoopConf gradoop config - * @throws Exception + * @throws Exception on failure */ private static void executeWithDemoData(GradoopFlinkConfig gradoopConf) throws Exception { diff --git a/gradoop-examples/src/main/java/org/gradoop/examples/utils/ExampleOutput.java b/gradoop-examples/src/main/java/org/gradoop/examples/utils/ExampleOutput.java index 077e56c10335..4451d0010204 100644 --- a/gradoop-examples/src/main/java/org/gradoop/examples/utils/ExampleOutput.java +++ b/gradoop-examples/src/main/java/org/gradoop/examples/utils/ExampleOutput.java @@ -79,7 +79,8 @@ public void add(String caption, GraphCollection collection) { /** * print output - * @throws Exception + * + * @throws Exception on failure */ public void print() throws Exception { outSet @@ -110,7 +111,7 @@ private static class LineCombiner implements MapFunction, String> { @Override - public String map(ArrayList lines) throws Exception { + public String map(ArrayList lines) { return StringUtils.join(lines, "\n"); } diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/converter/GradoopFormatConverter.java b/gradoop-examples/src/main/java/org/gradoop/utils/converter/GradoopFormatConverter.java index 796dd20e8b65..6e6f8712d71b 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/converter/GradoopFormatConverter.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/converter/GradoopFormatConverter.java @@ -33,6 +33,7 @@ public class GradoopFormatConverter extends AbstractRunner implements ProgramDes * args[3] - format of output graph (csv, json, indexed) * * @param args arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { LogicalGraph graph = readLogicalGraph(args[0], args[1]); diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/PageRankSamplingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/PageRankSamplingRunner.java index 44d074383e22..741e92af9f0a 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/PageRankSamplingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/PageRankSamplingRunner.java @@ -42,6 +42,7 @@ public class PageRankSamplingRunner extends AbstractRunner implements ProgramDes * * * @param args arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { LogicalGraph graph = readLogicalGraph(args[0], args[1]); diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomEdgeSamplingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomEdgeSamplingRunner.java index 0f21f6b1339a..1e90f8435bd2 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomEdgeSamplingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomEdgeSamplingRunner.java @@ -37,6 +37,7 @@ public class RandomEdgeSamplingRunner extends AbstractRunner implements ProgramD * args[4] - sampling threshold * * @param args arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { LogicalGraph graph = readLogicalGraph(args[0], args[1]); diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomLimitedDegreeVertexSamplingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomLimitedDegreeVertexSamplingRunner.java index b2406b2d8a25..1da25196a2a3 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomLimitedDegreeVertexSamplingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomLimitedDegreeVertexSamplingRunner.java @@ -41,6 +41,7 @@ public class RandomLimitedDegreeVertexSamplingRunner extends AbstractRunner impl * args[6] - vertex degree type (IN, OUT, BOTH) * * @param args arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { LogicalGraph graph = readLogicalGraph(args[0], args[1]); diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomNonUniformVertexSamplingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomNonUniformVertexSamplingRunner.java index 9b37d3f3c48b..eb4519add7c7 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomNonUniformVertexSamplingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomNonUniformVertexSamplingRunner.java @@ -37,6 +37,7 @@ public class RandomNonUniformVertexSamplingRunner extends AbstractRunner impleme * args[4] - sampling threshold * * @param args arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { LogicalGraph graph = readLogicalGraph(args[0], args[1]); diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexEdgeSamplingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexEdgeSamplingRunner.java index 76eda50ca581..801d6203854e 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexEdgeSamplingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexEdgeSamplingRunner.java @@ -39,6 +39,7 @@ public class RandomVertexEdgeSamplingRunner extends AbstractRunner implements Pr * args[6] - sampling type (SimpleVersion, NonuniformVersion, NonuniformHybridVersion) * * @param args arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { RandomVertexEdgeSampling.VertexEdgeSamplingType vertexEdgeSamplingType = diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexNeighborhoodSamplingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexNeighborhoodSamplingRunner.java index c6e7572e1111..8da5827c1bb2 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexNeighborhoodSamplingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexNeighborhoodSamplingRunner.java @@ -39,6 +39,7 @@ public class RandomVertexNeighborhoodSamplingRunner extends AbstractRunner imple * args[5] - type of neighborhood (IN, OUT, BOTH) * * @param args arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { LogicalGraph graph = readLogicalGraph(args[0], args[1]); diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexSamplingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexSamplingRunner.java index 8b60b8702b3b..db42c018d325 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexSamplingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomVertexSamplingRunner.java @@ -37,6 +37,7 @@ public class RandomVertexSamplingRunner extends AbstractRunner implements Progra * args[4] - sampling threshold * * @param args arguments + * @throws Exception on failure */ public static void main(String[] args) throws Exception { LogicalGraph graph = readLogicalGraph(args[0], args[1]); diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomWalkSamplingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomWalkSamplingRunner.java new file mode 100644 index 000000000000..ae945c6921fe --- /dev/null +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/RandomWalkSamplingRunner.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.utils.sampling; + +import org.apache.flink.api.common.ProgramDescription; +import org.gradoop.examples.AbstractRunner; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.sampling.RandomWalkSampling; + +/** + * Runs {@link RandomWalkSampling} for a given + * graph and writes the sampled output. + */ +public class RandomWalkSamplingRunner extends AbstractRunner implements ProgramDescription { + /** + * Runs the {@link RandomWalkSampling} operator + * on the specified graph and writes the result to the specified output. + * + * args[0] - path to input graph + * args[1] - format of input graph (csv, json, indexed) + * args[2] - sample size + * args[3] - number of start vertices + * args[4] - jump probability + * args[5] - max iteration + * args[6] - path to output graph + * args[7] - format of output graph (csv, json, indexed) + * + * @param args arguments + * @throws Exception on failure + */ + public static void main(String[] args) throws Exception { + + LogicalGraph graph = readLogicalGraph(args[0], args[1]); + + LogicalGraph sampledGraph = graph.callForGraph(new RandomWalkSampling( + Float.parseFloat(args[2]), Integer.parseInt(args[3]), + Float.parseFloat(args[4]), Integer.parseInt(args[5]))); + + writeLogicalGraph(sampledGraph, args[6], args[7]); + } + + @Override + public String getDescription() { + return RandomWalkSamplingRunner.class.getName(); + } +} diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/SamplingStatisticsRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/SamplingStatisticsRunner.java similarity index 77% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/SamplingStatisticsRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/sampling/SamplingStatisticsRunner.java index 98d2e50e8278..cb7d6cc8613f 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/SamplingStatisticsRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/sampling/SamplingStatisticsRunner.java @@ -13,10 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.sampling; import org.apache.flink.api.common.ProgramDescription; import org.gradoop.examples.AbstractRunner; +import org.gradoop.utils.statistics.AverageClusteringCoefficientRunner; +import org.gradoop.utils.statistics.AverageDegreeRunner; +import org.gradoop.utils.statistics.AverageIncomingDegreeRunner; +import org.gradoop.utils.statistics.AverageOutgoingDegreeRunner; +import org.gradoop.utils.statistics.ConnectedComponentsDistributionRunner; +import org.gradoop.utils.statistics.DegreeCentralityRunner; +import org.gradoop.utils.statistics.GlobalClusteringCoefficientRunner; +import org.gradoop.utils.statistics.GraphDensityRunner; +import org.gradoop.utils.statistics.TriangleCountingRunner; import org.gradoop.utils.statistics.VertexDegreeDistributionRunner; import org.gradoop.utils.statistics.VertexIncomingDegreeDistributionRunner; import org.gradoop.utils.statistics.VertexOutgoingDegreeDistributionRunner; @@ -52,6 +61,7 @@ public static void main(String[] args) throws Exception { AverageClusteringCoefficientRunner.main(args); GlobalClusteringCoefficientRunner.main(args); TriangleCountingRunner.main(args); + DegreeCentralityRunner.main(args); } @Override diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageClusteringCoefficientRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageClusteringCoefficientRunner.java similarity index 91% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageClusteringCoefficientRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageClusteringCoefficientRunner.java index 1e1818cb43a3..c0858cd820c4 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageClusteringCoefficientRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageClusteringCoefficientRunner.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.statistics; import org.apache.flink.api.common.ProgramDescription; import org.apache.flink.api.java.DataSet; @@ -21,8 +21,8 @@ import org.gradoop.flink.algorithms.gelly.clusteringcoefficient.GellyLocalClusteringCoefficientDirected; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.tuple.ObjectTo1; -import org.gradoop.flink.model.impl.operators.sampling.statistics.AverageClusteringCoefficient; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.AverageClusteringCoefficient; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; /** diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageDegreeRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageDegreeRunner.java similarity index 91% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageDegreeRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageDegreeRunner.java index 658c25b3389e..8f48fccd2849 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageDegreeRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageDegreeRunner.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.statistics; import org.apache.flink.api.common.ProgramDescription; import org.apache.flink.api.java.DataSet; import org.gradoop.examples.AbstractRunner; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.tuple.ObjectTo1; -import org.gradoop.flink.model.impl.operators.sampling.statistics.AverageDegree; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.AverageDegree; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; /** diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageIncomingDegreeRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageIncomingDegreeRunner.java similarity index 91% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageIncomingDegreeRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageIncomingDegreeRunner.java index a764b367694b..eff7e2352ef4 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageIncomingDegreeRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageIncomingDegreeRunner.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.statistics; import org.apache.flink.api.common.ProgramDescription; import org.apache.flink.api.java.DataSet; import org.gradoop.examples.AbstractRunner; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.tuple.ObjectTo1; -import org.gradoop.flink.model.impl.operators.sampling.statistics.AverageIncomingDegree; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.AverageIncomingDegree; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; /** diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageOutgoingDegreeRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageOutgoingDegreeRunner.java similarity index 91% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageOutgoingDegreeRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageOutgoingDegreeRunner.java index 3dec77cbf64f..d77483762ab9 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/AverageOutgoingDegreeRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/AverageOutgoingDegreeRunner.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.statistics; import org.apache.flink.api.common.ProgramDescription; import org.apache.flink.api.java.DataSet; import org.gradoop.examples.AbstractRunner; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.tuple.ObjectTo1; -import org.gradoop.flink.model.impl.operators.sampling.statistics.AverageOutgoingDegree; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.AverageOutgoingDegree; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; /** diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/ConnectedComponentsDistributionRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/ConnectedComponentsDistributionRunner.java similarity index 93% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/ConnectedComponentsDistributionRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/statistics/ConnectedComponentsDistributionRunner.java index 449b47c93e14..c280ef78113d 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/ConnectedComponentsDistributionRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/ConnectedComponentsDistributionRunner.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.statistics; import org.apache.flink.api.common.ProgramDescription; import org.gradoop.examples.AbstractRunner; -import org.gradoop.flink.model.impl.operators.sampling.statistics.ConnectedComponentsDistribution; +import org.gradoop.flink.model.impl.operators.statistics.ConnectedComponentsDistribution; import org.gradoop.flink.model.impl.epgm.LogicalGraph; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; /** diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/statistics/DegreeCentralityRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/DegreeCentralityRunner.java new file mode 100644 index 000000000000..49d811274058 --- /dev/null +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/DegreeCentralityRunner.java @@ -0,0 +1,63 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.utils.statistics; + +import org.apache.flink.api.common.ProgramDescription; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.tuple.Tuple1; +import org.gradoop.examples.AbstractRunner; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.functions.tuple.ObjectTo1; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.DegreeCentrality; +import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; + +/** + * Computes {@link DegreeCentrality} for a given logical graph. + * It writes the result as a single value into an csv-file: + * {@value SamplingEvaluationConstants#FILE_DEGREE_CENTRALITY} + */ +public class DegreeCentralityRunner extends AbstractRunner implements ProgramDescription { + + /** + * Calls the {@link DegreeCentrality} computation for a given logical graph. + * + * args[0] - path to input directory + * args[1] - input format (json, csv, indexed) + * args[2] - path to output directory + * + * @param args arguments + * @throws Exception if something goes wrong + */ + public static void main(String[] args) throws Exception { + + LogicalGraph logicalGraph = readLogicalGraph(args[0], args[1]); + + DataSet> resultSet = new DegreeCentrality() + .execute(logicalGraph) + .map(new ObjectTo1<>()); + + StatisticWriter.writeCSV(resultSet, + appendSeparator(args[2]) + SamplingEvaluationConstants.FILE_DEGREE_CENTRALITY); + + getExecutionEnvironment().execute("Statistics: Degree centrality"); + } + + @Override + public String getDescription() { + return DegreeCentralityRunner.class.getName(); + } +} diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/GlobalClusteringCoefficientRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/GlobalClusteringCoefficientRunner.java similarity index 95% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/GlobalClusteringCoefficientRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/statistics/GlobalClusteringCoefficientRunner.java index 33f295df61ae..48b64ad7d40c 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/GlobalClusteringCoefficientRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/GlobalClusteringCoefficientRunner.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.statistics; import org.apache.flink.api.common.ProgramDescription; import org.apache.flink.api.java.DataSet; @@ -22,7 +22,7 @@ import org.gradoop.flink.algorithms.gelly.clusteringcoefficient.GellyGlobalClusteringCoefficientDirected; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.tuple.ObjectTo1; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; /** diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/GraphDensityRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/GraphDensityRunner.java similarity index 91% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/GraphDensityRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/statistics/GraphDensityRunner.java index 12a4d028d25b..bed1ee7ac92a 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/GraphDensityRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/GraphDensityRunner.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.statistics; import org.apache.flink.api.common.ProgramDescription; import org.apache.flink.api.java.DataSet; import org.gradoop.examples.AbstractRunner; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.tuple.ObjectTo1; -import org.gradoop.flink.model.impl.operators.sampling.statistics.GraphDensity; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.GraphDensity; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; /** diff --git a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/TriangleCountingRunner.java b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/TriangleCountingRunner.java similarity index 94% rename from gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/TriangleCountingRunner.java rename to gradoop-examples/src/main/java/org/gradoop/utils/statistics/TriangleCountingRunner.java index d16da60e64ff..74e3e935ec3c 100644 --- a/gradoop-examples/src/main/java/org/gradoop/utils/sampling/statistics/TriangleCountingRunner.java +++ b/gradoop-examples/src/main/java/org/gradoop/utils/statistics/TriangleCountingRunner.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.utils.sampling.statistics; +package org.gradoop.utils.statistics; import org.apache.flink.api.common.ProgramDescription; import org.apache.flink.api.java.DataSet; @@ -21,7 +21,7 @@ import org.gradoop.flink.algorithms.gelly.trianglecounting.GellyTriangleCounting; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.tuple.ObjectTo1; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.gradoop.flink.model.impl.operators.statistics.writer.StatisticWriter; /** diff --git a/gradoop-examples/src/main/resources/META-INF/NOTICE b/gradoop-examples/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..08ad74d56384 --- /dev/null +++ b/gradoop-examples/src/main/resources/META-INF/NOTICE @@ -0,0 +1,19 @@ +gradoop-examples +Copyright 2014-2019 Leipzig University (Database Research Group) + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.github.s1ck:gdl:0.3 +- com.google.guava:guava +- commons-cli:commons-cli:1.4 +- log4j:log4j:1.2.17 +- me.lemire.integercompression:JavaFastPFOR:0.1.10 +- org.apache.flink:flink-clients_2.11:1.7.0 +- org.apache.flink:flink-hadoop-compatibility_2.11:1.7.0 +- org.apache.flink:flink-java:1.7.0 +- org.apache.flink:flink-shaded-hadoop2:1.7.0 +- org.codehaus.jettison:jettison:1.3.7 + +This project bundles the following dependencies under the BSD license. + +- org.antlr:antlr4-runtime diff --git a/gradoop-examples/src/main/resources/META-INF/licenses/LICENSE.antlr4 b/gradoop-examples/src/main/resources/META-INF/licenses/LICENSE.antlr4 new file mode 100644 index 000000000000..e00ceeefb8c6 --- /dev/null +++ b/gradoop-examples/src/main/resources/META-INF/licenses/LICENSE.antlr4 @@ -0,0 +1,10 @@ +Copyright (c) 2012 Terence Parr and Sam Harwell +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/gradoop-flink/pom.xml b/gradoop-flink/pom.xml index 9b1f01e92659..c7292d5fff61 100644 --- a/gradoop-flink/pom.xml +++ b/gradoop-flink/pom.xml @@ -1,11 +1,12 @@ - + 4.0.0 org.gradoop gradoop-parent - 0.4.4 + 0.4.5 gradoop-flink @@ -153,6 +154,17 @@ org.apache.flink flink-shaded-hadoop2 + + + + + org.opencypher + flink-cypher + + + + org.opencypher + spark-cypher @@ -195,4 +207,4 @@ - \ No newline at end of file + diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/BusinessTransactionGraphs.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/BusinessTransactionGraphs.java index cb2afe335648..9d5ddad43f76 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/BusinessTransactionGraphs.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/BusinessTransactionGraphs.java @@ -147,9 +147,4 @@ public GraphCollection execute(LogicalGraph iig) { return iig.getConfig().getGraphCollectionFactory() .fromDataSets(graphHeads, transVertices.union(masterVertices), btgEdges); } - - @Override - public String getName() { - return BusinessTransactionGraphs.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/BtgMessenger.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/BtgMessenger.java index 5837224e21a3..ad0d8bd40719 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/BtgMessenger.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/BtgMessenger.java @@ -28,7 +28,7 @@ public class BtgMessenger @Override public void sendMessages( - Vertex vertex) throws Exception { + Vertex vertex) { sendMessageToAllNeighbors(vertex.getValue()); } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/BtgUpdater.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/BtgUpdater.java index 0c75c781b7c1..a87a50d837dd 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/BtgUpdater.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/BtgUpdater.java @@ -28,7 +28,7 @@ public class BtgUpdater @Override public void updateVertex(Vertex vertex, - MessageIterator messageIterator) throws Exception { + MessageIterator messageIterator) { GradoopId lastComponent = vertex.getValue(); GradoopId newComponent = lastComponent; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/CollectGradoopIds.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/CollectGradoopIds.java index 298fac9d1ce0..2e475a0fcdda 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/CollectGradoopIds.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/CollectGradoopIds.java @@ -52,7 +52,7 @@ public void reduce(Iterable> mappings, @Override public void combine(Iterable> mappings, - Collector> collector) throws Exception { + Collector> collector) { boolean first = true; GradoopId vertexId = null; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/NewBtgGraphHead.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/NewBtgGraphHead.java index fcc957f0503f..17ab7d288b61 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/NewBtgGraphHead.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/btgs/functions/NewBtgGraphHead.java @@ -54,9 +54,6 @@ public G map(GradoopId graphId) throws Exception { return graphHead; } - /** - * {@inheritDoc} - */ @SuppressWarnings("unchecked") @Override public TypeInformation getProducedType() { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/fsm/transactional/CategoryCharacteristicSubgraphs.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/fsm/transactional/CategoryCharacteristicSubgraphs.java index 0ef0fa348706..cc2f199fe526 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/fsm/transactional/CategoryCharacteristicSubgraphs.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/fsm/transactional/CategoryCharacteristicSubgraphs.java @@ -248,9 +248,4 @@ private DataSet getCategoryFrequentSubgraphs( .withBroadcastSet(categoryMinFrequencies, DIMSpanConstants.MIN_FREQUENCY); } - @Override - public String getName() { - return this.getClass().getSimpleName(); - } - } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/fsm/transactional/ThinkLikeAnEmbeddingTFSM.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/fsm/transactional/ThinkLikeAnEmbeddingTFSM.java index 7bc2ad0d5f19..77d678bfaab6 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/fsm/transactional/ThinkLikeAnEmbeddingTFSM.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/fsm/transactional/ThinkLikeAnEmbeddingTFSM.java @@ -125,9 +125,4 @@ private DataSet getFrequentSubgraphs( .withBroadcastSet(minFrequency, DIMSpanConstants.MIN_FREQUENCY); } - @Override - public String getName() { - return this.getClass().getSimpleName(); - } - } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/BaseGellyAlgorithm.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/BaseGellyAlgorithm.java index 3240e5ea1632..1e4378043360 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/BaseGellyAlgorithm.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/BaseGellyAlgorithm.java @@ -29,9 +29,6 @@ */ public abstract class BaseGellyAlgorithm implements UnaryGraphToValueOperator { - /** - * {@inheritDoc} - */ @Override public O execute(LogicalGraph graph) { try { @@ -54,6 +51,7 @@ public O execute(LogicalGraph graph) { * * @param graph The Gelly graph. * @return output format. + * @throws Exception on failure */ public abstract O executeInGelly(Graph graph) throws Exception; } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/GradoopGellyAlgorithm.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/GradoopGellyAlgorithm.java index 84d65a612f01..f29cb33fb930 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/GradoopGellyAlgorithm.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/GradoopGellyAlgorithm.java @@ -62,9 +62,6 @@ protected GradoopGellyAlgorithm( this.toGellyEdge = edgeValue; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph execute(LogicalGraph graph) { this.currentGraph = graph; @@ -94,6 +91,7 @@ public Graph transformToGelly(LogicalGraph graph) { * * @param graph The Gelly graph. * @return The Gradoop graph. + * @throws Exception on failure */ public abstract LogicalGraph executeInGelly(Graph graph) throws Exception; } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/ClusteringCoefficientBase.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/ClusteringCoefficientBase.java index df57a5fc6c6e..caf873277471 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/ClusteringCoefficientBase.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/ClusteringCoefficientBase.java @@ -72,9 +72,4 @@ public LogicalGraph executeInGelly(Graph graph) */ protected abstract LogicalGraph executeInternal(Graph gellyGraph) throws Exception; - - @Override - public String getName() { - return ClusteringCoefficientBase.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyGlobalClusteringCoefficientDirected.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyGlobalClusteringCoefficientDirected.java index 2d3957761b8e..0c4e86f6c5ed 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyGlobalClusteringCoefficientDirected.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyGlobalClusteringCoefficientDirected.java @@ -64,9 +64,4 @@ protected LogicalGraph executeInternal(Graph ge return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( resultHead, currentGraph.getVertices(), currentGraph.getEdges()); } - - @Override - public String getName() { - return GellyGlobalClusteringCoefficientDirected.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyGlobalClusteringCoefficientUndirected.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyGlobalClusteringCoefficientUndirected.java index 47062a698c8f..977a82b0f3bd 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyGlobalClusteringCoefficientUndirected.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyGlobalClusteringCoefficientUndirected.java @@ -65,9 +65,4 @@ protected LogicalGraph executeInternal(Graph ge return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( resultHead, currentGraph.getVertices(), currentGraph.getEdges()); } - - @Override - public String getName() { - return GellyGlobalClusteringCoefficientUndirected.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyLocalClusteringCoefficientDirected.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyLocalClusteringCoefficientDirected.java index 92db361e0e83..71151329c869 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyLocalClusteringCoefficientDirected.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyLocalClusteringCoefficientDirected.java @@ -60,9 +60,4 @@ protected LogicalGraph executeInternal(Graph ge return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( currentGraph.getGraphHead(), resultVertices, currentGraph.getEdges()); } - - @Override - public String getName() { - return GellyLocalClusteringCoefficientDirected.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyLocalClusteringCoefficientUndirected.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyLocalClusteringCoefficientUndirected.java index c268e9ad1ea9..6c14a6c12286 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyLocalClusteringCoefficientUndirected.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyLocalClusteringCoefficientUndirected.java @@ -61,9 +61,4 @@ protected LogicalGraph executeInternal(Graph ge return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( currentGraph.getGraphHead(), resultVertices, currentGraph.getEdges()); } - - @Override - public String getName() { - return GellyLocalClusteringCoefficientUndirected.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/AnnotateWeaklyConnectedComponents.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/AnnotateWeaklyConnectedComponents.java index 0b38273ca728..216b26e69566 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/AnnotateWeaklyConnectedComponents.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/AnnotateWeaklyConnectedComponents.java @@ -107,9 +107,4 @@ public LogicalGraph executeInGelly(Graph graph) return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( currentGraph.getGraphHead(), annotatedVertices, edges); } - - @Override - public String getName() { - return AnnotateWeaklyConnectedComponents.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/ValueWeaklyConnectedComponents.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/ValueWeaklyConnectedComponents.java index e8e197aae290..0b78d21b0660 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/ValueWeaklyConnectedComponents.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/ValueWeaklyConnectedComponents.java @@ -99,12 +99,4 @@ public DataSet> executeInGelly(Graph g return new ConnectedComponents(maxIteration) .run(graph).map(new MapVertexIdComponentId()); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return ValueWeaklyConnectedComponents.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/WeaklyConnectedComponentsAsCollection.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/WeaklyConnectedComponentsAsCollection.java index 6a97ff604d46..3a4d66531cba 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/WeaklyConnectedComponentsAsCollection.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/WeaklyConnectedComponentsAsCollection.java @@ -78,9 +78,4 @@ public GraphCollection execute(LogicalGraph graph) { split.getVertices().map(new PropertyRemover<>(propertyKey)), split.getEdges().map(new PropertyRemover<>(propertyKey))); } - - @Override - public String getName() { - return WeaklyConnectedComponentsAsCollection.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/functions/MapVertexIdComponentId.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/functions/MapVertexIdComponentId.java index 229fc9a95ef6..c7b2c7e11b07 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/functions/MapVertexIdComponentId.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/connectedcomponents/functions/MapVertexIdComponentId.java @@ -24,9 +24,6 @@ */ public class MapVertexIdComponentId implements MapFunction, Tuple2> { - /** - * {@inheritDoc} - */ @Override public Tuple2 map(Vertex gellyVertex) throws Exception { return gellyVertex; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/hits/HITS.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/hits/HITS.java index 633f2a36dfc2..a9fd5aafd000 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/hits/HITS.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/hits/HITS.java @@ -114,9 +114,4 @@ public LogicalGraph executeInGelly(Graph graph) .fromDataSets(newVertices, currentGraph.getEdges()); } - @Override - public String getName() { - return HITS.class.getName(); - } - } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/hits/functions/HitsResultKeySelector.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/hits/functions/HitsResultKeySelector.java index a8728fd4e113..e43aefccc259 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/hits/functions/HitsResultKeySelector.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/hits/functions/HitsResultKeySelector.java @@ -31,7 +31,7 @@ public class HitsResultKeySelector * * @param gradoopIdResult HITS Algorithm result * @return selects gradoop id - * @throws Exception + * @throws Exception on failure */ @Override public GradoopId getKey(HITS.Result gradoopIdResult) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/labelpropagation/LabelPropagation.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/labelpropagation/LabelPropagation.java index 33bc06aca893..6101e8980aeb 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/labelpropagation/LabelPropagation.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/labelpropagation/LabelPropagation.java @@ -68,9 +68,6 @@ protected LabelPropagation(int maxIterations, String propertyKey) { this.propertyKey = checkNotNull(propertyKey); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph executeInGelly(Graph graph) { DataSet labeledVertices = executeInternal(graph) @@ -100,12 +97,4 @@ public LogicalGraph executeInGelly(Graph gr protected int getMaxIterations() { return maxIterations; } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return LabelPropagation.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/labelpropagation/functions/LPUpdateFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/labelpropagation/functions/LPUpdateFunction.java index 843626965a5c..076f9bf1ec0c 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/labelpropagation/functions/LPUpdateFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/labelpropagation/functions/LPUpdateFunction.java @@ -36,11 +36,10 @@ public class LPUpdateFunction * * @param vertex vertex to be updated * @param msg message - * @throws Exception */ @Override public void updateVertex(Vertex vertex, - MessageIterator msg) throws Exception { + MessageIterator msg) { PropertyValue value = getNewValue(vertex, Lists.newArrayList(msg.iterator())); if (!vertex.getValue().equals(value)) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/pagerank/PageRank.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/pagerank/PageRank.java index b84a26c8a591..4f84704195f9 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/pagerank/PageRank.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/pagerank/PageRank.java @@ -99,12 +99,4 @@ public LogicalGraph executeInGelly(Graph graph) return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( currentGraph.getGraphHead(), newVertices, currentGraph.getEdges()); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return PageRank.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/KRandomJumpGellyVCI.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/KRandomJumpGellyVCI.java new file mode 100644 index 000000000000..ddc8b1351cf9 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/KRandomJumpGellyVCI.java @@ -0,0 +1,229 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump; + +import org.apache.flink.api.common.aggregators.LongSumAggregator; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.utils.DataSetUtils; +import org.apache.flink.graph.Edge; +import org.apache.flink.graph.Graph; +import org.apache.flink.graph.Vertex; +import org.apache.flink.graph.pregel.VertexCentricConfiguration; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.flink.algorithms.gelly.BaseGellyAlgorithm; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.EPGMEdgeWithGellyEdgeIdJoin; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.GellyVertexWithEPGMVertexJoin; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.GellyVertexWithLongIdToGradoopIdJoin; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.GetVisitedGellyEdgeLongIdsFlatMap; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.GetVisitedSourceTargetIdsFlatMap; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.LongIdToGellyVertexWithVCIValueMap; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.LongIdTupleToGellyEdgeWithLongValueJoin; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.LongIdWithEdgeToTupleJoin; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.ReplaceTargetWithLongIdJoin; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.VCIComputeFunction; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.VCIVertexValue; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.VertexWithVisitedSourceTargetIdJoin; +import org.gradoop.flink.algorithms.gelly.randomjump.functions.VisitedGellyEdgesWithLongIdToGradoopIdJoin; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.functions.epgm.Id; +import org.gradoop.flink.model.impl.functions.epgm.SourceId; +import org.gradoop.flink.model.impl.functions.tuple.Value0Of2; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; + +import java.util.HashSet; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * Performs the RandomJump using Gellys VertexCentricIteration (VCI). + * Uniformly at random picks {@link #numberOfStartVertices} starting vertices and then simulates a RandomWalk for each + * starting vertex on the graph, where once visited edges are not used again. For each walker, + * with a given {@link #jumpProbability}, or if the walk ends in a sink, or if all outgoing edges + * of the current vertex were visited, randomly jumps to any vertex in the graph and starts a new + * RandomWalk from there. + * Unlike the RandomWalk algorithm, RandomJump does not have problems of getting stuck or not being + * able to visit enough vertices. The algorithm converges when the maximum number of iterations + * has been reached, or enough vertices have been visited (with the percentage of vertices to + * visit at least given in {@link #percentageToVisit}). + * Returns the initial graph with vertices and edges annotated by a boolean property named + * "sampled", which is set to {@code true} if visited, or {@code false} if not. + */ +public class KRandomJumpGellyVCI + extends BaseGellyAlgorithm { + /** + * The graph used in {@link KRandomJumpGellyVCI#execute(LogicalGraph)}. + */ + protected LogicalGraph currentGraph; + + /** + * Number of starting vertices. + */ + private final int numberOfStartVertices; + + /** + * Value for maximum number of iterations for the algorithm. + */ + private final int maxIterations; + + /** + * Probability for jumping to a random vertex instead of walking to a random neighbor. + */ + private final double jumpProbability; + + /** + * Relative amount of vertices to visit at least. + */ + private final double percentageToVisit; + + /** + * DataSet holding the mapping for a long index to its vertex gradoop id. + */ + private DataSet> indexToVertexIdMap; + + /** + * DataSet holding the mapping for a long index to its edge gradoop id. + */ + private DataSet> indexToEdgeIdMap; + + /** + * Creates an instance of KRandomJumpGellyVCI. + * + * @param numberOfStartVertices Number of starting vertices. + * @param maxIterations Value for maximum number of iterations for the algorithm. + * @param jumpProbability Probability for jumping to random vertex instead of walking to random + * neighbor. + * @param percentageToVisit Relative amount of vertices to visit at least. + */ + public KRandomJumpGellyVCI(int numberOfStartVertices, int maxIterations, double jumpProbability, + double percentageToVisit) { + checkArgument(numberOfStartVertices >= 1, + "at least 1 starting vertex is needed, numberOfStartVertices must be equal or greater 1"); + checkArgument(maxIterations > 0, + "maxIterations must be greater than 0"); + checkArgument(jumpProbability >= 0d && jumpProbability <= 1d, + "jumpProbability must be equal/greater than 0.0 and smaller/equal 1.0"); + checkArgument(percentageToVisit > 0d && percentageToVisit <= 1d, + "percentageToVisit must be greater than 0.0 and smaller/equal 1.0"); + this.numberOfStartVertices = numberOfStartVertices; + this.maxIterations = maxIterations; + this.jumpProbability = jumpProbability; + this.percentageToVisit = percentageToVisit; + } + + @Override + public Graph transformToGelly(LogicalGraph graph) { + this.currentGraph = graph; + + indexToVertexIdMap = DataSetUtils.zipWithIndex(graph.getVertices().map(new Id<>())); + indexToEdgeIdMap = DataSetUtils.zipWithIndex(graph.getEdges().map(new Id<>())); + + DataSet> vertices = indexToVertexIdMap + .map(new LongIdToGellyVertexWithVCIValueMap()); + + DataSet> edges = graph.getEdges() + .join(indexToVertexIdMap) + .where(new SourceId<>()).equalTo(1) + .with(new LongIdWithEdgeToTupleJoin()) + .join(indexToVertexIdMap) + .where(1).equalTo(1) + .with(new ReplaceTargetWithLongIdJoin()) + .join(indexToEdgeIdMap) + .where(2).equalTo(1) + .with(new LongIdTupleToGellyEdgeWithLongValueJoin()); + + return Graph.fromDataSet(vertices, edges, graph.getConfig().getExecutionEnvironment()); + } + + @Override + public LogicalGraph executeInGelly(Graph gellyGraph) + throws Exception { + + Long vertexCount = gellyGraph.numberOfVertices(); + + //-------------------------------------------------------------------------- + // pre compute + //-------------------------------------------------------------------------- + + // define start vertices + Set randomStartIndices = new HashSet<>(); + while (randomStartIndices.size() < numberOfStartVertices) { + long randomLongInBounds = (long) (Math.random() * (vertexCount - 1L)); + randomStartIndices.add(randomLongInBounds); + } + DataSet startIndices = currentGraph.getConfig().getExecutionEnvironment() + .fromCollection(randomStartIndices); + + // define how many vertices to visit + long verticesToVisit = (long) Math.ceil((double) vertexCount * percentageToVisit); + + // set compute parameters + VertexCentricConfiguration parameters = new VertexCentricConfiguration(); + parameters.addBroadcastSet(VCIComputeFunction.START_INDICES_BROADCAST_SET, startIndices); + parameters.addBroadcastSet(VCIComputeFunction.VERTEX_INDICES_BROADCAST_SET, + indexToVertexIdMap.map(new Value0Of2<>())); + parameters.registerAggregator(VCIComputeFunction.VISITED_VERTICES_AGGREGATOR_NAME, + new LongSumAggregator()); + + // run gelly + Graph resultGraph = gellyGraph.runVertexCentricIteration( + new VCIComputeFunction(jumpProbability, verticesToVisit), + null, maxIterations, parameters); + + //-------------------------------------------------------------------------- + // post compute + //-------------------------------------------------------------------------- + + DataSet visitedGellyEdgeIds = resultGraph.getVertices() + .flatMap(new GetVisitedGellyEdgeLongIdsFlatMap()) + .join(indexToEdgeIdMap) + .where("*").equalTo(0) + .with(new VisitedGellyEdgesWithLongIdToGradoopIdJoin()); + + // compute new visited edges + DataSet visitedEdges = currentGraph.getEdges() + .leftOuterJoin(visitedGellyEdgeIds) + .where(new Id<>()).equalTo("*") + .with(new EPGMEdgeWithGellyEdgeIdJoin(SamplingConstants.PROPERTY_KEY_SAMPLED)); + + DataSet visitedSourceTargetIds = visitedEdges + .flatMap(new GetVisitedSourceTargetIdsFlatMap(SamplingConstants.PROPERTY_KEY_SAMPLED)) + .distinct(); + + // compute new visited vertices + DataSet visitedVertices = resultGraph.getVertices() + .join(indexToVertexIdMap) + .where(0).equalTo(0) + .with(new GellyVertexWithLongIdToGradoopIdJoin()) + .join(currentGraph.getVertices()) + .where(0).equalTo(new Id<>()) + .with(new GellyVertexWithEPGMVertexJoin(SamplingConstants.PROPERTY_KEY_SAMPLED)); + + visitedVertices = visitedVertices.leftOuterJoin(visitedSourceTargetIds) + .where(new Id<>()).equalTo("*") + .with(new VertexWithVisitedSourceTargetIdJoin(SamplingConstants.PROPERTY_KEY_SAMPLED)); + + // return graph + return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( + currentGraph.getGraphHead(), visitedVertices, visitedEdges); + } + + @Override + public String getName() { + return KRandomJumpGellyVCI.class.getName(); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/EPGMEdgeWithGellyEdgeIdJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/EPGMEdgeWithGellyEdgeIdJoin.java new file mode 100644 index 000000000000..f18850293d82 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/EPGMEdgeWithGellyEdgeIdJoin.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; + +/** + * Joins an EPGM edge with a GradoopId of an visited edge from the VCI run. Assigns a boolean + * property to the EPGM edge, determining if this edge was visited. The property is set to + * {@code true} if the EPGM edge has a join partner, set to {@code false} otherwise. + */ +@FunctionAnnotation.ForwardedFieldsFirst("id;sourceId;targetId;label;graphIds") +public class EPGMEdgeWithGellyEdgeIdJoin implements JoinFunction { + + /** + * Key for the boolean property of the edge. + */ + private final String propertyKey; + + /** + * Creates an instance of EPGMEdgeWithGellyEdgeIdJoin with a given key for the boolean + * property value. + * + * @param propertyKey propertyKey Key for the boolean property value. + */ + public EPGMEdgeWithGellyEdgeIdJoin(String propertyKey) { + this.propertyKey = propertyKey; + } + + @Override + public Edge join(Edge edge, GradoopId visitedEdgeId) throws Exception { + boolean visited = false; + if (visitedEdgeId != null) { + visited = true; + } + edge.setProperty(propertyKey, visited); + return edge; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GellyVertexWithEPGMVertexJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GellyVertexWithEPGMVertexJoin.java new file mode 100644 index 000000000000..0f815688cfb2 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GellyVertexWithEPGMVertexJoin.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Vertex; + +/** + * Joins a Gelly result vertex with an EPGM vertex. Assigning a boolean property value from the + * Gelly vertex to the EPGM vertex, determining if this vertex was visited. + */ +@FunctionAnnotation.ReadFieldsFirst("f1") +@FunctionAnnotation.ForwardedFieldsSecond("id;label;graphIds") +public class GellyVertexWithEPGMVertexJoin implements + JoinFunction, Vertex, Vertex> { + + /** + * Key for the boolean property value to assign to the EPGM vertex. + */ + private final String propertyKey; + + /** + * Creates an instance of GellyVertexWithEPGMVertexJoin with a given key for + * the boolean property value. + * + * @param propertyKey Key for the boolean property value. + */ + public GellyVertexWithEPGMVertexJoin(String propertyKey) { + this.propertyKey = propertyKey; + } + + @Override + public Vertex join(org.apache.flink.graph.Vertex gellyVertex, + Vertex epgmVertex) throws Exception { + epgmVertex.setProperty(propertyKey, gellyVertex.getValue().f0); + return epgmVertex; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GellyVertexWithLongIdToGradoopIdJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GellyVertexWithLongIdToGradoopIdJoin.java new file mode 100644 index 000000000000..a3d4a885c85c --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GellyVertexWithLongIdToGradoopIdJoin.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.graph.Vertex; +import org.gradoop.common.model.impl.id.GradoopId; + +/** + * Joins a Gelly vertex with its index-to-GradoopId-mapping to replace the index with the GradoopId. + */ +@FunctionAnnotation.ForwardedFieldsFirst("f1->f1") +@FunctionAnnotation.ForwardedFieldsSecond("f1->f0") +public class GellyVertexWithLongIdToGradoopIdJoin implements + JoinFunction, Tuple2, + Vertex> { + + /** + * Reduce object instantiation. + */ + private final Vertex reuseVertex; + + /** + * Creates an instance of GellyVertexWithLongIdToGradoopIdJoin. + */ + public GellyVertexWithLongIdToGradoopIdJoin() { + this.reuseVertex = new Vertex<>(); + } + + @Override + public Vertex join( + Vertex gellyVertexWithLongId, Tuple2 indexToGradoopId) + throws Exception { + reuseVertex.setId(indexToGradoopId.f1); + reuseVertex.setValue(gellyVertexWithLongId.getValue()); + return reuseVertex; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GetVisitedGellyEdgeLongIdsFlatMap.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GetVisitedGellyEdgeLongIdsFlatMap.java new file mode 100644 index 000000000000..07c2d3cae6ef --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GetVisitedGellyEdgeLongIdsFlatMap.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.graph.Vertex; +import org.apache.flink.util.Collector; + +/** + * Retrieves the long index for all visited outgoing edges from a gelly source vertex. + */ +@FunctionAnnotation.ReadFields("f1") +public class GetVisitedGellyEdgeLongIdsFlatMap implements + FlatMapFunction, Long> { + + @Override + public void flatMap(Vertex gellyVertex, + Collector out) throws Exception { + for (Long visitedOutEdge : gellyVertex.getValue().getVisitedOutEdges()) { + out.collect(visitedOutEdge); + } + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GetVisitedSourceTargetIdsFlatMap.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GetVisitedSourceTargetIdsFlatMap.java new file mode 100644 index 000000000000..b9b2d77d6f3f --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/GetVisitedSourceTargetIdsFlatMap.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; + +/** + * Retrieves the source- and target-id from an edge, if this edge was visited, determined by the + * boolean property. + */ +@FunctionAnnotation.ReadFields("properties") +public class GetVisitedSourceTargetIdsFlatMap implements FlatMapFunction { + + /** + * Key for the boolean property value to determine, if the edge was visited. + */ + private final String propertyKey; + + /** + * Creates an instance of GetVisitedSourceTargetIdsFlatMap with a given property key. + * + * @param propertyKey Key for the boolean property value to determine, if the edge was visited. + */ + public GetVisitedSourceTargetIdsFlatMap(String propertyKey) { + this.propertyKey = propertyKey; + } + + @Override + public void flatMap(Edge edge, Collector out) throws Exception { + if (edge.getPropertyValue(propertyKey).getBoolean()) { + out.collect(edge.getSourceId()); + out.collect(edge.getTargetId()); + } + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdToGellyVertexWithVCIValueMap.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdToGellyVertexWithVCIValueMap.java new file mode 100644 index 000000000000..e88695a38216 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdToGellyVertexWithVCIValueMap.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.graph.Vertex; +import org.gradoop.common.model.impl.id.GradoopId; + +import java.util.ArrayList; + +/** + * Creates a Gelly vertex with an unique long id as id and a {@link VCIVertexValue} as value. + */ +@FunctionAnnotation.ForwardedFields("f0") +public class LongIdToGellyVertexWithVCIValueMap implements + MapFunction, Vertex> { + + /** + * Reduce object instantiation. + */ + private final Vertex reuseVertex; + + /** + * Reduce object instantiation. + */ + private final VCIVertexValue reuseValue; + + /** + * Creates an instance of LongIdToGellyVertexWithVCIValueMap. + */ + public LongIdToGellyVertexWithVCIValueMap() { + this.reuseVertex = new Vertex<>(); + this.reuseValue = new VCIVertexValue(false, new ArrayList<>()); + } + + @Override + public Vertex map(Tuple2 tuple) throws Exception { + reuseVertex.setId(tuple.f0); + reuseVertex.setValue(reuseValue); + return reuseVertex; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdTupleToGellyEdgeWithLongValueJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdTupleToGellyEdgeWithLongValueJoin.java new file mode 100644 index 000000000000..a90ee63c3d04 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdTupleToGellyEdgeWithLongValueJoin.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.graph.Edge; +import org.gradoop.common.model.impl.id.GradoopId; + +/** + * Creates a Gelly edge with its long index as its value. + */ +@FunctionAnnotation.ForwardedFieldsFirst("f0;f1") +@FunctionAnnotation.ForwardedFieldsSecond("f0->f2") +public class LongIdTupleToGellyEdgeWithLongValueJoin implements + JoinFunction, Tuple2, Edge> { + + /** + * Reduce object instantiation. + */ + private final Edge reuseEdge; + + /** + * Creates an instance of LongIdTupleToGellyEdgeWithLongValueJoin. + */ + public LongIdTupleToGellyEdgeWithLongValueJoin() { + this.reuseEdge = new Edge<>(); + } + + @Override + public Edge join(Tuple3 edgeTuple, + Tuple2 indexToEdgeId) throws Exception { + reuseEdge.setSource(edgeTuple.f0); + reuseEdge.setTarget(edgeTuple.f1); + reuseEdge.setValue(indexToEdgeId.f0); + return reuseEdge; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdWithEdgeToTupleJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdWithEdgeToTupleJoin.java new file mode 100644 index 000000000000..a886f7967b77 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/LongIdWithEdgeToTupleJoin.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple3; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; + +/** + * Takes an EPGM edge and creates a tuple, containing the edges source id as unique long id, its + * target id as GradoopId and its own id as GradoopId. + */ +@FunctionAnnotation.ForwardedFieldsFirst("targetId->f1;id->f2") +@FunctionAnnotation.ForwardedFieldsSecond("f0") +public class LongIdWithEdgeToTupleJoin implements + JoinFunction, Tuple3> { + + /** + * Reduce object instantiation. + */ + private final Tuple3 reuseTuple; + + /** + * Creates an instance of LongIdWithEdgeToTupleJoin. + */ + public LongIdWithEdgeToTupleJoin() { + this.reuseTuple = new Tuple3<>(); + } + + @Override + public Tuple3 join( + Edge epgmEdge, + Tuple2 uniqueLongToVertexId) throws Exception { + reuseTuple.f0 = uniqueLongToVertexId.f0; + reuseTuple.f1 = epgmEdge.getTargetId(); + reuseTuple.f2 = epgmEdge.getId(); + return reuseTuple; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/ReplaceTargetWithLongIdJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/ReplaceTargetWithLongIdJoin.java new file mode 100644 index 000000000000..b05259ca51fe --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/ReplaceTargetWithLongIdJoin.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple3; +import org.gradoop.common.model.impl.id.GradoopId; + +/** + * Replaces the target GradoopId with the unique long id this target vertex is mapped to. + */ +@FunctionAnnotation.ForwardedFieldsFirst("f0;f2") +@FunctionAnnotation.ForwardedFieldsSecond("f0->f1") +public class ReplaceTargetWithLongIdJoin implements + JoinFunction, Tuple2, + Tuple3> { + + /** + * Reduce object instantiation. + */ + private final Tuple3 reuseTuple; + + /** + * Creates an instance of ReplaceTargetWithLongIdJoin. + */ + public ReplaceTargetWithLongIdJoin() { + this.reuseTuple = new Tuple3<>(); + } + + @Override + public Tuple3 join(Tuple3 edgeTuple, + Tuple2 uniqueLongToVertexId) throws Exception { + reuseTuple.f0 = edgeTuple.f0; + reuseTuple.f1 = uniqueLongToVertexId.f0; + reuseTuple.f2 = edgeTuple.f2; + return reuseTuple; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VCIComputeFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VCIComputeFunction.java new file mode 100644 index 000000000000..775fd41009fb --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VCIComputeFunction.java @@ -0,0 +1,203 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import com.google.common.collect.Lists; +import org.apache.flink.api.common.aggregators.LongSumAggregator; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.graph.Edge; +import org.apache.flink.graph.Vertex; +import org.apache.flink.graph.pregel.ComputeFunction; +import org.apache.flink.graph.pregel.MessageIterator; +import org.apache.flink.types.LongValue; +import org.apache.flink.types.NullValue; +import org.gradoop.flink.algorithms.gelly.randomjump.KRandomJumpGellyVCI; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Compute function for {@link KRandomJumpGellyVCI}. Vertex values are of type + * {@link VCIVertexValue}, with fields: + *

    + *
  • f0: {@code Boolean} set to {@code true} if the vertex was visited, to {@code false} + * otherwise + *
  • f1: {@code List} containing all long indices from already visited outgoing edges + *

+ * A message of type {@code NullValue} from one vertex to another is a walk respective a jump to + * this other vertex and therefor considered as one of {@link KRandomJumpGellyVCI#k} walkers. + */ +public class VCIComputeFunction extends ComputeFunction { + + /** + * Name of the broadcast set containing the indices for the k starting vertices. + */ + public static final String START_INDICES_BROADCAST_SET = "startIndices"; + + /** + * Name of the broadcast set containing the graphs vertex indices. + */ + public static final String VERTEX_INDICES_BROADCAST_SET = "vertexIndices"; + + /** + * Name for the LongSumAggregator used for counting the visited vertices. + */ + public static final String VISITED_VERTICES_AGGREGATOR_NAME = "visitedVerticesAggregator"; + + /** + * Probability for jumping to a random vertex instead of walking to a random neighbor. + */ + private final double jumpProbability; + + /** + * Number of vertices to visit at least. + */ + private final long verticesToVisit; + + /** + * List with the indices for the k starting vertices. + */ + private List startIndices; + + /** + * List containing the graphs vertex indices. + */ + private List vertexIndices; + + /** + * The LongSumAggregator used for counting the visited vertices. + */ + private LongSumAggregator visitedVerticesAggregator; + + /** + * Keeping track of the currently visited vertices at the beginning at each superstep. + */ + private long currentVisitedCount; + + /** + * Creates an instance of VCIComputeFunction. + * + * @param jumpProbability Probability for jumping to random vertex instead of walking to random + * neighbor. + * @param verticesToVisit Number of vertices to visit via walk or jump. + */ + public VCIComputeFunction(double jumpProbability, long verticesToVisit) { + this.jumpProbability = jumpProbability; + this.verticesToVisit = verticesToVisit; + this.visitedVerticesAggregator = new LongSumAggregator(); + this.currentVisitedCount = 0L; + } + + /** + * {@inheritDoc} + * + * Reads the broadcast sets for the starting vertices and the graph vertices. Retrieves the + * aggregator for visited vertices and accumulates the visited vertices from the previous + * superstep. + */ + @Override + public void preSuperstep() { + startIndices = (List) this.getBroadcastSet(START_INDICES_BROADCAST_SET); + vertexIndices = (List) this.getBroadcastSet(VERTEX_INDICES_BROADCAST_SET); + visitedVerticesAggregator = getIterationAggregator(VISITED_VERTICES_AGGREGATOR_NAME); + LongValue previousAggregate = getPreviousIterationAggregate(VISITED_VERTICES_AGGREGATOR_NAME); + if (previousAggregate != null) { + currentVisitedCount += previousAggregate.getValue(); + } + } + + /** + * {@inheritDoc} + * + * Initially starts the first walk or jump from the given start vertices. Afterwards starts a + * walk or jump from a vertex, if it received messages from other vertices. Stops the + * computation and therefor the iteration if the number of currently visited vertices exceeds + * the number of vertices to visit. + * + * @param vertex The vertex this computation in a superstep is running for. + * @param messages Iterator over all incoming messages + */ + @Override + public void compute(Vertex vertex, MessageIterator messages) { + if (currentVisitedCount < verticesToVisit) { + List> edgesList = Lists.newArrayList(getEdges()); + Tuple2 valueWithHasChanged = Tuple2.of(vertex.getValue(), false); + if (startIndices.contains(vertex.getId()) && !valueWithHasChanged.f0.isVisited()) { + valueWithHasChanged = walkToRandomNeighbor(valueWithHasChanged, edgesList); + } else if (messages.hasNext()) { + for (NullValue msg : messages) { + valueWithHasChanged = walkToRandomNeighbor(valueWithHasChanged, edgesList); + } + } + if (valueWithHasChanged.f1) { + setNewVertexValue(valueWithHasChanged.f0); + } + } + } + + /** + * Performs a walk to a random neighbor by sending a message to a target from an unvisited + * outgoing edge. Sets the vertex value as visited if necessary and updates the visited edge ids. + * Sets a boolean flag, if the vertex value has changed. + * Alternatively performs a jump to a random vertex with a probability given in + * {@link #jumpProbability} or if there are no unvisited outgoing edges. + * Returns the vertex value with the boolean flag as {@code Tuple2} eventually. + * + * @param valueWithHasChanged {@code Tuple2} containing the vertex value and a boolean flag + * determining if the value has changed + * @param edgesList List of all outgoing edge for the vertex + * @return {@code Tuple2} containing the vertex value and a boolean flag determining if the + * value has changed + */ + private Tuple2 walkToRandomNeighbor( + Tuple2 valueWithHasChanged, List> edgesList) { + if (!valueWithHasChanged.f0.isVisited()) { + visitedVerticesAggregator.aggregate(1L); + valueWithHasChanged.f0.setVisited(); + valueWithHasChanged.f1 = true; + } + if ((jumpProbability == 0d) || (jumpProbability < ThreadLocalRandom.current().nextDouble())) { + List> unvisitedNeighborWithEdgeId = new ArrayList<>(); + for (Edge edge : edgesList) { + if (!valueWithHasChanged.f0.getVisitedOutEdges().contains(edge.getValue())) { + unvisitedNeighborWithEdgeId.add(Tuple2.of(edge.getTarget(), edge.getValue())); + } + } + if (!unvisitedNeighborWithEdgeId.isEmpty()) { + int randomIndex = ThreadLocalRandom.current().nextInt(unvisitedNeighborWithEdgeId.size()); + Long randomNeighborIndex = unvisitedNeighborWithEdgeId.get(randomIndex).f0; + valueWithHasChanged.f0.addVisitedOutEdge(unvisitedNeighborWithEdgeId.get(randomIndex).f1); + sendMessageTo(randomNeighborIndex, new NullValue()); + valueWithHasChanged.f1 = true; + } else { + jumpToRandomVertex(); + } + } else { + jumpToRandomVertex(); + } + return valueWithHasChanged; + } + + /** + * Jumps to a random vertex in the graph by sending a message to this vertex. + */ + private void jumpToRandomVertex() { + int randomIndex = ThreadLocalRandom.current().nextInt(vertexIndices.size()); + Long randomVertexIndex = vertexIndices.get(randomIndex); + sendMessageTo(randomVertexIndex, new NullValue()); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VCIVertexValue.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VCIVertexValue.java new file mode 100644 index 000000000000..f4ab932ea39a --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VCIVertexValue.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.java.tuple.Tuple2; + +import java.util.List; + +/** + * The vertex value used for the Gelly vertex centric iteration. + */ +public class VCIVertexValue extends Tuple2> { + + /** + * Creates an empty instance of VCIVertexValue. + */ + public VCIVertexValue() { } + + /** + * Creates an instance of VCIVertexValue with the given values. + * + * @param visited {@code Boolean} determining if a vertex was visited ({@code true}) + * or not ({@code false}). + * @param visitedOutEdgeIds {@code List} containing the ids of all visited outgoing edges. + */ + VCIVertexValue(Boolean visited, List visitedOutEdgeIds) { + super(visited, visitedOutEdgeIds); + } + + /** + * Checks if the vertex was visited. + * + * @return {@code true} if it was visited, {@code false} otherwise. + */ + public boolean isVisited() { + return this.f0; + } + + /** + * Sets the vertex as visited. + */ + public void setVisited() { + this.f0 = true; + } + + /** + * Gets all visited outgoing edges. + * + * @return List containing the ids off all visited outgoing edges. + */ + public List getVisitedOutEdges() { + return this.f1; + } + + /** + * Adds an id to the list of visited outgoing edges. + * + * @param edgeId The newly visited outgoing edge id. + */ + public void addVisitedOutEdge(Long edgeId) { + this.f1.add(edgeId); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VertexWithVisitedSourceTargetIdJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VertexWithVisitedSourceTargetIdJoin.java new file mode 100644 index 000000000000..45567f5bef42 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VertexWithVisitedSourceTargetIdJoin.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Vertex; + +/** + * Joins an EPGM vertex with the source- resp. target-id of a visited edge. Sets the vertex visited + * property to {@code true}, if there is a join-partner. + */ +@FunctionAnnotation.ForwardedFieldsFirst("id;label;graphIds") +public class VertexWithVisitedSourceTargetIdJoin implements JoinFunction { + + /** + * Key for the boolean property of the edge. + */ + private final String propertyKey; + + /** + * Creates an instance of VertexWithVisitedSourceTargetIdJoin with a given property key. + * + * @param propertyKey propertyKey Key for the boolean property value. + */ + public VertexWithVisitedSourceTargetIdJoin(String propertyKey) { + this.propertyKey = propertyKey; + } + + @Override + public Vertex join(Vertex vertex, GradoopId visitedId) throws Exception { + if (visitedId != null) { + vertex.setProperty(propertyKey, true); + } + return vertex; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VisitedGellyEdgesWithLongIdToGradoopIdJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VisitedGellyEdgesWithLongIdToGradoopIdJoin.java new file mode 100644 index 000000000000..ad54c89faf55 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/VisitedGellyEdgesWithLongIdToGradoopIdJoin.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.id.GradoopId; + +/** + * Joins an edge index with its index-to-GradoopId-mapping to replace the index with the GradoopId. + */ +@FunctionAnnotation.ForwardedFieldsSecond("f1->*") +public class VisitedGellyEdgesWithLongIdToGradoopIdJoin implements + JoinFunction, GradoopId> { + + @Override + public GradoopId join(Long edgeLongId, Tuple2 indexToEdgeId) throws Exception { + return indexToEdgeId.f1; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/package-info.java new file mode 100644 index 000000000000..cc3b00d604ba --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/functions/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains user defined functions related to the Random Jump algorithm. + */ +package org.gradoop.flink.algorithms.gelly.randomjump.functions; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/package-info.java new file mode 100644 index 000000000000..257ab85e0832 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/randomjump/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains classes related to the Random Jump algorithm. + */ +package org.gradoop.flink.algorithms.gelly.randomjump; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/shortestpaths/SingleSourceShortestPaths.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/shortestpaths/SingleSourceShortestPaths.java index 28af16ef2d5b..39a6d0c22c18 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/shortestpaths/SingleSourceShortestPaths.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/shortestpaths/SingleSourceShortestPaths.java @@ -86,9 +86,4 @@ public LogicalGraph executeInGelly(Graph graph) return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets(newVertices, currentGraph.getEdges()); } - - @Override - public String getName() { - return SingleSourceShortestPaths.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/trianglecounting/GellyTriangleCounting.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/trianglecounting/GellyTriangleCounting.java index 9fc643af4e7d..6d299d3db329 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/trianglecounting/GellyTriangleCounting.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/trianglecounting/GellyTriangleCounting.java @@ -65,9 +65,4 @@ public LogicalGraph executeInGelly(Graph graph) return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( resultHead, currentGraph.getVertices(), currentGraph.getEdges()); } - - @Override - public String getName() { - return GellyTriangleCounting.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/vertexdegrees/DistinctVertexDegrees.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/vertexdegrees/DistinctVertexDegrees.java index 1c0bb8cc3db1..c0da93da9475 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/vertexdegrees/DistinctVertexDegrees.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/vertexdegrees/DistinctVertexDegrees.java @@ -99,9 +99,4 @@ public LogicalGraph executeInGelly(Graph graph) return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets(newVertices, currentGraph.getEdges()); } - - @Override - public String getName() { - return DistinctVertexDegrees.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/vertexdegrees/functions/DistinctVertexDegreesToAttribute.java b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/vertexdegrees/functions/DistinctVertexDegreesToAttribute.java index 75c79db484fc..f62685af0b91 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/vertexdegrees/functions/DistinctVertexDegreesToAttribute.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/algorithms/gelly/vertexdegrees/functions/DistinctVertexDegreesToAttribute.java @@ -25,46 +25,45 @@ /** * Stores the in-degree, out-degree and the sum of both as a property in vertex */ -public class DistinctVertexDegreesToAttribute implements JoinFunction, Vertex, Vertex> { +public class DistinctVertexDegreesToAttribute + implements JoinFunction + , Vertex, Vertex> { /** * Property to store the sum of vertex degrees in. */ - private final String vertexDegreesPropery; + private final String vertexDegreesProperty; /** * Property to store the in vertex degree in. */ - private final String vertexInDegreePropery; + private final String vertexInDegreeProperty; /** * Property to store the out vertex degree in. */ - private final String vertexOutDegreePropery; + private final String vertexOutDegreeProperty; /** * Stores the in, out and sum of in and out degrees of a vertex. * - * @param vertexDegreesPropery property key to store sum degree - * @param vertexInDegreesPropery property key to store in degree - * @param vertexOutDegreesPropery property key to store out degree + * @param vertexDegreesProperty property key to store sum degree + * @param vertexInDegreesProperty property key to store in degree + * @param vertexOutDegreesProperty property key to store out degree */ - public DistinctVertexDegreesToAttribute(String vertexDegreesPropery, - String vertexInDegreesPropery, String vertexOutDegreesPropery) { - this.vertexDegreesPropery = vertexDegreesPropery; - this.vertexInDegreePropery = vertexInDegreesPropery; - this.vertexOutDegreePropery = vertexOutDegreesPropery; + public DistinctVertexDegreesToAttribute(String vertexDegreesProperty, + String vertexInDegreesProperty, String vertexOutDegreesProperty) { + this.vertexDegreesProperty = vertexDegreesProperty; + this.vertexInDegreeProperty = vertexInDegreesProperty; + this.vertexOutDegreeProperty = vertexOutDegreesProperty; } @Override public Vertex join(org.apache.flink.graph.Vertex degree, Vertex vertex) throws Exception { - vertex.setProperty( - vertexDegreesPropery, + vertex.setProperty(vertexDegreesProperty, PropertyValue.create(degree.getValue().getDegree().getValue())); - vertex.setProperty( - vertexInDegreePropery, + vertex.setProperty(vertexInDegreeProperty, PropertyValue.create(degree.getValue().getInDegree().getValue())); - vertex.setProperty( - vertexOutDegreePropery, + vertex.setProperty(vertexOutDegreeProperty, PropertyValue.create(degree.getValue().getOutDegree().getValue())); return vertex; } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/api/DataSink.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/api/DataSink.java index 2853627ba455..83b09e17e31b 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/api/DataSink.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/api/DataSink.java @@ -29,6 +29,7 @@ public interface DataSink { * Writes a logical graph to the data sink. * * @param logicalGraph logical graph + * @throws IOException on failure */ void write(LogicalGraph logicalGraph) throws IOException; @@ -36,6 +37,7 @@ public interface DataSink { * Writes a graph collection graph to the data sink. * * @param graphCollection graph collection + * @throws IOException on failure */ void write(GraphCollection graphCollection) throws IOException; @@ -44,6 +46,7 @@ public interface DataSink { * * @param logicalGraph logical graph * @param overwrite true, if existing files should be overwritten + * @throws IOException on failure */ void write(LogicalGraph logicalGraph, boolean overwrite) throws IOException; @@ -52,6 +55,7 @@ public interface DataSink { * * @param graphCollection graph collection * @param overwrite true, if existing files should be overwritten + * @throws IOException on failure */ void write(GraphCollection graphCollection, boolean overwrite) throws IOException; } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/api/DataSource.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/api/DataSource.java index f7734c388a1f..6b51ca05af0c 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/api/DataSource.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/api/DataSource.java @@ -28,7 +28,8 @@ public interface DataSource { /** * Reads the input as logical graph. * - * @return logial graph + * @return logical graph + * @throws IOException on failure */ LogicalGraph getLogicalGraph() throws IOException; @@ -36,6 +37,7 @@ public interface DataSource { * Reads the input as graph collection. * * @return graph collection + * @throws IOException on failure */ GraphCollection getGraphCollection() throws IOException; } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/api/metadata/MetaDataSource.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/api/metadata/MetaDataSource.java index 26c5e6c0c018..ff5de062a649 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/api/metadata/MetaDataSource.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/api/metadata/MetaDataSource.java @@ -128,6 +128,7 @@ static DataSet> tuplesFromEle * @param path path to metadata csv file * @param hdfsConfig file system configuration * @return meta data + * @throws IOException on failure */ M readLocal(String path, Configuration hdfsConfig) throws IOException; } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/CSVDataSource.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/CSVDataSource.java index 3e83d8bb793d..b6bfb8a14a7b 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/CSVDataSource.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/CSVDataSource.java @@ -27,7 +27,6 @@ import org.gradoop.flink.io.impl.csv.metadata.CSVMetaDataSource; import org.gradoop.flink.model.impl.epgm.GraphCollection; import org.gradoop.flink.model.impl.epgm.LogicalGraph; -import org.gradoop.flink.model.impl.operators.combination.ReduceCombination; import org.gradoop.flink.util.GradoopFlinkConfig; /** @@ -55,14 +54,17 @@ public CSVDataSource(String csvPath, GradoopFlinkConfig config) { } /** + * Will use a single graph head of the collection as final graph head for the graph. + * Issue #1217 (https://github.com/dbs-leipzig/gradoop/issues/1217) will optimize further. + * * {@inheritDoc} - *

- * Graph heads will be disposed at the moment. The following issue attempts to provide - * alternatives to keep graph heads: https://github.com/dbs-leipzig/gradoop/issues/974 */ @Override public LogicalGraph getLogicalGraph() { - return getGraphCollection().reduce(new ReduceCombination()); + GraphCollection collection = getGraphCollection(); + return getConfig().getLogicalGraphFactory() + .fromDataSets( + collection.getGraphHeads().first(1), collection.getVertices(), collection.getEdges()); } @Override diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/indexed/IndexedCSVDataSource.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/indexed/IndexedCSVDataSource.java index b550b4b01483..fa5d1ae4a8d6 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/indexed/IndexedCSVDataSource.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/indexed/IndexedCSVDataSource.java @@ -66,9 +66,8 @@ public class IndexedCSVDataSource extends CSVBase implements DataSource { * * @param csvPath root path of csv files * @param config gradoop configuration - * @throws IOException on failure */ - public IndexedCSVDataSource(String csvPath, GradoopFlinkConfig config) throws IOException { + public IndexedCSVDataSource(String csvPath, GradoopFlinkConfig config) { this(csvPath, config, new Configuration()); } @@ -78,7 +77,6 @@ public IndexedCSVDataSource(String csvPath, GradoopFlinkConfig config) throws IO * @param csvPath root path of csv files * @param conf gradoop configuration * @param hdfsConf HDFS configuration - * @throws IOException on failure */ public IndexedCSVDataSource(String csvPath, GradoopFlinkConfig conf, Configuration hdfsConf) { super(csvPath, conf); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaData.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaData.java index 40b0af3a23a2..02db8c796c24 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaData.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaData.java @@ -36,7 +36,7 @@ public class CSVMetaData extends MetaData { * @param vertexMetaData a map between each vertex label and its property metadata * @param edgeMetaData a map between each edge label and its property metadata */ - CSVMetaData( + public CSVMetaData( Map> graphMetaData, Map> vertexMetaData, Map> edgeMetaData) { @@ -59,8 +59,8 @@ public List getPropertyMetaData(String type, String label) { case MetaDataSource.EDGE_TYPE: return this.edgeMetaData.getOrDefault(label, new ArrayList<>()); default: - throw new IllegalArgumentException("Entity type " + type + " is not supported. Supported" + - "types are g,v and e."); + throw new IllegalArgumentException("Entity type " + type + " is not supported. Supported " + + "types are g, v and e."); } } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaDataParser.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaDataParser.java index bac4eb9e97a7..d9e22824b263 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaDataParser.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaDataParser.java @@ -39,49 +39,32 @@ */ public class CSVMetaDataParser { /** - * Used to map a type string to its corresponding parsing function + * Used to map a simple type string to its corresponding parsing function. */ - private static final Map> TYPE_PARSER_MAP = getTypeParserMap(); + private static final Map> SIMPLE_TYPE_PARSER_MAP = + getSimpleTypeParserMap(); /** - * Creates the type - parser function mapping of static property TYPE_PARSER_MAP + * Creates the type - parser function mapping of static property SIMPLE_TYPE_PARSER_MAP * - * @return a HashMap containing the mapping of a type string to its corresponding parsing function + * @return a HashMap containing the mapping of a simple type string to its corresponding + * parsing function. */ - private static Map> getTypeParserMap() { + private static Map> getSimpleTypeParserMap() { Map> map = new HashMap<>(); - map.put(MetaData.TypeString.SHORT.getTypeString(), - Short::parseShort); - map.put(MetaData.TypeString.INTEGER.getTypeString(), - Integer::parseInt); - map.put(MetaData.TypeString.LONG.getTypeString(), - Long::parseLong); - map.put(MetaData.TypeString.FLOAT.getTypeString(), - Float::parseFloat); - map.put(MetaData.TypeString.DOUBLE.getTypeString(), - Double::parseDouble); - map.put(MetaData.TypeString.BOOLEAN.getTypeString(), - Boolean::parseBoolean); - map.put(MetaData.TypeString.STRING.getTypeString(), - CSVMetaDataParser::parseStringProperty); - map.put(MetaData.TypeString.BIGDECIMAL.getTypeString(), - BigDecimal::new); - map.put(MetaData.TypeString.GRADOOPID.getTypeString(), - GradoopId::fromString); - map.put(MetaData.TypeString.MAP.getTypeString(), - CSVMetaDataParser::parseMapProperty); - map.put(MetaData.TypeString.LIST.getTypeString(), - CSVMetaDataParser::parseListProperty); - map.put(MetaData.TypeString.LOCALDATE.getTypeString(), - LocalDate::parse); - map.put(MetaData.TypeString.LOCALTIME.getTypeString(), - LocalTime::parse); - map.put(MetaData.TypeString.LOCALDATETIME.getTypeString(), - LocalDateTime::parse); - map.put(MetaData.TypeString.NULL.getTypeString(), - CSVMetaDataParser::parseNullProperty); - map.put(MetaData.TypeString.SET.getTypeString(), - CSVMetaDataParser::parseSetProperty); + map.put(MetaData.TypeString.SHORT.getTypeString(), Short::parseShort); + map.put(MetaData.TypeString.INTEGER.getTypeString(), Integer::parseInt); + map.put(MetaData.TypeString.LONG.getTypeString(), Long::parseLong); + map.put(MetaData.TypeString.FLOAT.getTypeString(), Float::parseFloat); + map.put(MetaData.TypeString.DOUBLE.getTypeString(), Double::parseDouble); + map.put(MetaData.TypeString.BOOLEAN.getTypeString(), Boolean::parseBoolean); + map.put(MetaData.TypeString.STRING.getTypeString(), StringEscaper::unescape); + map.put(MetaData.TypeString.BIGDECIMAL.getTypeString(), BigDecimal::new); + map.put(MetaData.TypeString.GRADOOPID.getTypeString(), GradoopId::fromString); + map.put(MetaData.TypeString.LOCALDATE.getTypeString(), LocalDate::parse); + map.put(MetaData.TypeString.LOCALTIME.getTypeString(), LocalTime::parse); + map.put(MetaData.TypeString.LOCALDATETIME.getTypeString(), LocalDateTime::parse); + map.put(MetaData.TypeString.NULL.getTypeString(), CSVMetaDataParser::parseNullProperty); return Collections.unmodifiableMap(map); } @@ -106,114 +89,94 @@ public static String getPropertyMetaData(Property property) { * @return parsing function for the specific type */ static Function getPropertyValueParser(String propertyType) { - String[] propertyTypeTokens = StringEscaper.split( - propertyType, PropertyMetaData.PROPERTY_TOKEN_DELIMITER); - if (propertyTypeTokens.length == 2 && - propertyTypeTokens[0].equals(MetaData.TypeString.LIST.getTypeString())) { - // It's a list with one additional data type (type of list items). - return getListValueParser(propertyTypeTokens[1]); - } else if (propertyTypeTokens.length == 2 && - propertyTypeTokens[0].equals(MetaData.TypeString.SET.getTypeString())) { - // It's a set with one additional data type (type of set items). - return getSetValueParser(propertyTypeTokens[1]); - } else if (propertyTypeTokens.length == 3 && - propertyTypeTokens[0].equals(MetaData.TypeString.MAP.getTypeString())) { - // It's a map with two additional data types (key type + value type). - return getMapValueParser(propertyTypeTokens[1], propertyTypeTokens[2]); + String[] typeTokens = StringEscaper.split( + propertyType.toLowerCase(), PropertyMetaData.PROPERTY_TOKEN_DELIMITER); + String mainType = typeTokens[0]; + + if (mainType.equals(MetaData.TypeString.LIST.getTypeString())) { + return getListValueParser(typeTokens); + } else if (mainType.equals(MetaData.TypeString.SET.getTypeString())) { + return getSetValueParser(typeTokens); + } else if (mainType.equals(MetaData.TypeString.MAP.getTypeString())) { + return getMapValueParser(typeTokens); + } else if (SIMPLE_TYPE_PARSER_MAP.containsKey(mainType)) { + return SIMPLE_TYPE_PARSER_MAP.get(mainType); } else { - return getValueParser(propertyType); - } - } - - /** - * Creates a parsing function for the given primitive property type. - * - * @param type property type - * @return parsing function - */ - private static Function getValueParser(String type) { - type = type.toLowerCase(); - if (TYPE_PARSER_MAP.containsKey(type)) { - return TYPE_PARSER_MAP.get(type); - } else { - throw new IllegalArgumentException("Type " + type + " is not supported"); + throw new TypeNotPresentException(mainType, null); } } /** * Creates a parsing function for list property type. * - * @param listItemType string representation of the list item type, e.g. "String" + * @param listTypeTokens string tokens of the list type and its items type, e.g. + * ["list", "string"] * @return parsing function */ - private static Function getListValueParser(String listItemType) { - final String itemType = listItemType.toLowerCase(); + private static Function getListValueParser(String[] listTypeTokens) { + // It's a list with one additional data type (type of list items). + if (listTypeTokens.length != 2) { + throw new IllegalArgumentException("Item type of List type is missing"); + } + final String itemType = listTypeTokens[1]; // check the validity of the list item type - if (!TYPE_PARSER_MAP.containsKey(itemType)) { + if (!SIMPLE_TYPE_PARSER_MAP.containsKey(itemType)) { throw new TypeNotPresentException(itemType, null); } - return s -> parseListProperty(s, TYPE_PARSER_MAP.get(itemType)); + return s -> parseListProperty(s, SIMPLE_TYPE_PARSER_MAP.get(itemType)); } /** * Creates a parsing function for map property type. * - * @param keyType string representation of the map key type, e.g. "String" - * @param valueType string representation of the map value type, e.g. "Double" + * @param mapTypeTokens string tokens of the map type and its key type and value type, e.g. + * ["map", "string", "double"] * @return parsing function */ - private static Function getMapValueParser(String keyType, String valueType) { - final String keyTypeLowerCase = keyType.toLowerCase(); + private static Function getMapValueParser(String[] mapTypeTokens) { + // It's a map with two additional data types (key type + value type). + if (mapTypeTokens.length != 3) { + throw new IllegalArgumentException("Key type or value type of Map type is missing"); + } + + final String keyType = mapTypeTokens[1]; // check the validity of the map key type - if (!TYPE_PARSER_MAP.containsKey(keyTypeLowerCase)) { - throw new TypeNotPresentException(keyTypeLowerCase, null); + if (!SIMPLE_TYPE_PARSER_MAP.containsKey(keyType)) { + throw new TypeNotPresentException(keyType, null); } - final String valueTypeLowerCase = valueType.toLowerCase(); + final String valueType = mapTypeTokens[2]; // check the validity of the map value type - if (!TYPE_PARSER_MAP.containsKey(valueTypeLowerCase)) { - throw new TypeNotPresentException(keyTypeLowerCase, null); + if (!SIMPLE_TYPE_PARSER_MAP.containsKey(valueType)) { + throw new TypeNotPresentException(valueType, null); } return s -> parseMapProperty( s, - TYPE_PARSER_MAP.get(keyTypeLowerCase), - TYPE_PARSER_MAP.get(valueTypeLowerCase) + SIMPLE_TYPE_PARSER_MAP.get(keyType), + SIMPLE_TYPE_PARSER_MAP.get(valueType) ); } /** * Creates a parsing function for set property type. * - * @param setItemType string representation of the set item type, e.g. "String" + * @param setTypeTokens string tokens of the set type and its item type, e.g. ["set", "string"] * @return parsing function */ - private static Function getSetValueParser(String setItemType) { - final String itemType = setItemType.toLowerCase(); + private static Function getSetValueParser(String[] setTypeTokens) { + // It's a set with one additional data type (type of set items). + if (setTypeTokens.length != 2) { + throw new IllegalArgumentException("Item type of Set type is missing"); + } + final String itemType = setTypeTokens[1]; // check the validity of the set item type - if (!TYPE_PARSER_MAP.containsKey(itemType)) { + if (!SIMPLE_TYPE_PARSER_MAP.containsKey(itemType)) { throw new TypeNotPresentException(itemType, null); } - return s -> parseSetProperty(s, TYPE_PARSER_MAP.get(itemType)); - } - - /** - * Parse function to translate string representation of a List to a list of PropertyValues - * Every PropertyValue has the type "string", because there is no parser for the items given - * Use {@link #parseListProperty(String, Function)} to specify a parsing function - * - * @param s the string to parse as list, e.g. "[myString1,myString2]" - * @return the list represented by the argument - */ - private static Object parseListProperty(String s) { - // no item type given, so use string as type - s = s.substring(1, s.length() - 1); - return Arrays.stream(StringEscaper.split(s, CSVConstants.LIST_DELIMITER)) - .map(StringEscaper::unescape) - .map(PropertyValue::create) - .collect(Collectors.toList()); + return s -> parseSetProperty(s, SIMPLE_TYPE_PARSER_MAP.get(itemType)); } /** @@ -231,24 +194,6 @@ private static Object parseListProperty(String s, Function itemP .collect(Collectors.toList()); } - /** - * Parse function to translate string representation of a Map to a Map with key and value of - * type PropertyValue. Every PropertyValue (key and value) has the type "string", because there - * are no parsers for the keys and values given. Use - * {@link #parseMapProperty(String, Function, Function)} to specify both parsing functions. - * - * @param s the string to parse as map, e.g. "{myString1=myValue1,myString2=myValue2}" - * @return the map represented by the argument - */ - private static Object parseMapProperty(String s) { - // no key type and value type given, so use string as types - s = s.substring(1, s.length() - 1); - return Arrays.stream(StringEscaper.split(s, CSVConstants.LIST_DELIMITER)) - .map(st -> StringEscaper.split(st, CSVConstants.MAP_SEPARATOR)) - .collect(Collectors.toMap(e -> PropertyValue.create(StringEscaper.unescape(e[0])), - e -> PropertyValue.create(StringEscaper.unescape(e[1])))); - } - /** * Parse function to translate string representation of a Map to a Map with * key and value of type PropertyValue. @@ -263,32 +208,10 @@ private static Object parseMapProperty(String s, Function keyPar s = s.substring(1, s.length() - 1); return Arrays.stream(StringEscaper.split(s, CSVConstants.LIST_DELIMITER)) .map(st -> StringEscaper.split(st, CSVConstants.MAP_SEPARATOR)) - .map(strings -> { - Object[] objects = new Object[2]; - objects[0] = keyParser.apply(strings[0]); - objects[1] = valueParser.apply(strings[1]); - return objects; - }) + .map(strings -> new Object[]{keyParser.apply(strings[0]), valueParser.apply(strings[1])}) .collect(Collectors.toMap(e -> PropertyValue.create(e[0]), e -> PropertyValue.create(e[1]))); } - /** - * Parse function to translate string representation of a Set to a set of PropertyValues. - * Every PropertyValue has the type "string", because there is no parser for the items given. - * Use {@link #parseListProperty(String, Function)} to specify a parsing function. - * - * @param s the string to parse as set, e.g. "[myString1,myString2]" - * @return the set represented by the argument - */ - private static Object parseSetProperty(String s) { - // no item type given, so use string as type - s = s.substring(1, s.length() - 1); - return Arrays.stream(StringEscaper.split(s, CSVConstants.LIST_DELIMITER)) - .map(StringEscaper::unescape) - .map(PropertyValue::create) - .collect(Collectors.toSet()); - } - /** * Parse function to translate string representation of a Set to a set of PropertyValues. * @@ -304,16 +227,6 @@ private static Object parseSetProperty(String s, Function itemPa .collect(Collectors.toSet()); } - /** - * Parse function to translate CSV strings to strings. - * - * @param s the string to parse - * @return the unescaped string - */ - private static Object parseStringProperty(String s) { - return StringEscaper.unescape(s); - } - /** * Parse function to create null from the null string representation. * diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaDataSource.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaDataSource.java index 65755c62dba0..40c250758528 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaDataSource.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/CSVMetaDataSource.java @@ -28,6 +28,7 @@ import org.gradoop.flink.util.GradoopFlinkConfig; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.Charset; @@ -103,7 +104,7 @@ public CSVMetaData readLocal(String path, Configuration hdfsConfig) throws IOExc Charset charset = Charset.forName("UTF-8"); if (!fs.exists(file)) { - return null; + throw new FileNotFoundException(path); } else { try (BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(file), charset))) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/EntityToJSON.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/EntityToJSON.java index 878e84d02bef..1784ca710d4b 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/EntityToJSON.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/EntityToJSON.java @@ -36,7 +36,7 @@ public class EntityToJSON { * * @param entity entity with key-value properties * @return json object containing the properties - * @throws JSONException + * @throws JSONException if failed to create JSON */ protected JSONObject writeProperties(EPGMAttributed entity) throws JSONException { @@ -57,7 +57,7 @@ protected JSONObject writeProperties(EPGMAttributed entity) throws * @param entity labeled graph element (e.g., vertex and edge) * @param input element type * @return json object containing meta information - * @throws JSONException + * @throws JSONException if failed to create JSON */ protected JSONObject writeGraphElementMeta( @@ -76,7 +76,7 @@ protected JSONObject writeProperties(EPGMAttributed entity) throws * @param entity logical graph data * @param graph data type * @return json object with graph meta data - * @throws JSONException + * @throws JSONException if failed to create JSON */ protected JSONObject writeGraphMeta(T entity) throws JSONException { @@ -88,7 +88,7 @@ protected JSONObject writeProperties(EPGMAttributed entity) throws * * @param entity labeled entity * @return json object with meta data containing the label - * @throws JSONException + * @throws JSONException if failed to create JSON */ private JSONObject writeMeta(EPGMLabeled entity) throws JSONException { JSONObject meta = new JSONObject(); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToEdge.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToEdge.java index 5c0d60508c08..01e42e3196f5 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToEdge.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToEdge.java @@ -62,7 +62,7 @@ public JSONToEdge(EPGMEdgeFactory epgmEdgeFactory) { * * @param s json string * @return Gelly edge storing gradoop edge data - * @throws Exception + * @throws Exception on failure */ @Override public Edge map(String s) throws Exception { @@ -84,7 +84,7 @@ public Edge map(String s) throws Exception { * * @param jsonEdge json string representation * @return source vertex identifier - * @throws JSONException + * @throws JSONException if failed to parse JSON */ private GradoopId getSourceId(JSONObject jsonEdge) throws JSONException { return GradoopId.fromString(jsonEdge.getString(JSONConstants.EDGE_SOURCE)); @@ -95,7 +95,7 @@ private GradoopId getSourceId(JSONObject jsonEdge) throws JSONException { * * @param jsonEdge json string representation * @return target vertex identifier - * @throws JSONException + * @throws JSONException if failed to parse JSON */ private GradoopId getTargetId(JSONObject jsonEdge) throws JSONException { return GradoopId.fromString(jsonEdge.getString(JSONConstants.EDGE_TARGET)); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToEntity.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToEntity.java index 702e735af1af..a39519d4a8b9 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToEntity.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToEntity.java @@ -36,7 +36,7 @@ public class JSONToEntity { * * @param object json object * @return entity identifier - * @throws JSONException + * @throws JSONException if failed to parse JSON */ protected GradoopId getID(JSONObject object) throws JSONException { return GradoopId.fromString(object.getString(JSONConstants.IDENTIFIER)); @@ -47,7 +47,7 @@ protected GradoopId getID(JSONObject object) throws JSONException { * * @param object json object * @return entity label - * @throws JSONException + * @throws JSONException if failed to parse JSON */ protected String getLabel(JSONObject object) throws JSONException { return object @@ -60,7 +60,7 @@ protected String getLabel(JSONObject object) throws JSONException { * * @param object json object * @return key-value properties - * @throws JSONException + * @throws JSONException if failed to parse JSON */ protected Map getProperties(JSONObject object) throws JSONException { @@ -81,7 +81,7 @@ protected Map getProperties(JSONObject object) throws * * @param object json object * @return graph identifiers - * @throws JSONException + * @throws JSONException if failed to parse JSON */ protected GradoopIdSet getGraphs(JSONObject object) throws JSONException { GradoopIdSet result; @@ -100,7 +100,7 @@ protected GradoopIdSet getGraphs(JSONObject object) throws JSONException { * * @param array json array * @return long values - * @throws JSONException + * @throws JSONException if failed to parse JSON */ protected GradoopIdSet getArrayValues(JSONArray array) throws JSONException { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToGraphHead.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToGraphHead.java index 2490ff341f45..a37f94c70160 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToGraphHead.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToGraphHead.java @@ -59,7 +59,7 @@ public JSONToGraphHead(EPGMGraphHeadFactory epgmGraphHeadFactory) { * * @param s json string representation * @return SubgraphWithCount storing graph data - * @throws Exception + * @throws Exception on failure */ @Override public GraphHead map(String s) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToVertex.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToVertex.java index d7b43cdb5869..9da13c593508 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToVertex.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/json/functions/JSONToVertex.java @@ -61,7 +61,7 @@ public JSONToVertex(EPGMVertexFactory epgmVertexFactory) { * * @param s json string * @return Gelly vertex storing gradoop vertex data - * @throws Exception + * @throws Exception on failure */ @Override public Vertex map(String s) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/logicalgraphcsv/MetaData.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/logicalgraphcsv/MetaData.java index b2036aa5c551..faedb59f5cc5 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/logicalgraphcsv/MetaData.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/deprecated/logicalgraphcsv/MetaData.java @@ -82,7 +82,7 @@ public static DataSet> fromFile(String path, Grad * @param path path to metadata csv file * @param hdfsConfig file system configuration * @return meta data - * @throws IOException + * @throws IOException on failure */ public static MetaData fromFile(String path, Configuration hdfsConfig) throws IOException { FileSystem fs = FileSystem.get(hdfsConfig); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/dot/DOTDataSink.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/dot/DOTDataSink.java index 46e218a3bd93..c70ca6da28d1 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/dot/DOTDataSink.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/dot/DOTDataSink.java @@ -99,6 +99,7 @@ private static class GraphvizWriter extends TextOutputFormat { /** * see super constructor. + * * @param outputPath graphviz dot file name * @param charset encoding */ @@ -108,24 +109,19 @@ private static class GraphvizWriter extends TextOutputFormat { /** * see super constructor. + * * @param outputPath graphviz dot file name */ GraphvizWriter(Path outputPath) { super(outputPath); } - /** - * {@inheritDoc} - */ @Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); super.writeRecord("digraph {\n"); } - /** - * {@inheritDoc} - */ @Override public void close() throws IOException { super.writeRecord("}"); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/edgelist/functions/CreateImportEdge.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/edgelist/functions/CreateImportEdge.java index fbe8da689251..3b2ecb66cbc3 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/edgelist/functions/CreateImportEdge.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/edgelist/functions/CreateImportEdge.java @@ -52,10 +52,9 @@ public CreateImportEdge() { /** * Method to create ImportEdge * - * @param idTuple tuple that contains unique line id + source and - * target ids - * @return initialized reuseEdge - * @throws Exception + * @param idTuple tuple that contains unique line id + source and target ids + * @return initialized reuseEdge + * @throws Exception on failure */ @Override public ImportEdge map(Tuple2> idTuple) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/gdl/GDLDataSink.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/gdl/GDLDataSink.java index 2902e27279be..c72f7f01357e 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/gdl/GDLDataSink.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/gdl/GDLDataSink.java @@ -45,33 +45,21 @@ public GDLDataSink(String path) { this.path = path; } - /** - * {@inheritDoc} - */ @Override public void write(LogicalGraph logicalGraph) throws IOException { write(logicalGraph, false); } - /** - * {@inheritDoc} - */ @Override public void write(GraphCollection graphCollection) throws IOException { write(graphCollection, false); } - /** - * {@inheritDoc} - */ @Override public void write(LogicalGraph logicalGraph, boolean overwrite) throws IOException { write(logicalGraph.getConfig().getGraphCollectionFactory().fromGraph(logicalGraph), overwrite); } - /** - * {@inheritDoc} - */ @Override public void write(GraphCollection graphCollection, boolean overwrite) throws IOException { FileSystem.WriteMode writeMode = diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/gdl/functions/GraphTransactionsToGDL.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/gdl/functions/GraphTransactionsToGDL.java index cd89aab702e4..81c44dcea64e 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/gdl/functions/GraphTransactionsToGDL.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/gdl/functions/GraphTransactionsToGDL.java @@ -31,9 +31,6 @@ */ public class GraphTransactionsToGDL implements GroupReduceFunction { - /** - * {@inheritDoc} - */ @Override public void reduce(Iterable graphTransactions, Collector out) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/InitEdge.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/InitEdge.java index e3ec8a9afa4a..47fb3c9031e1 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/InitEdge.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/InitEdge.java @@ -76,7 +76,7 @@ public InitEdge(EPGMEdgeFactory epgmEdgeFactory, String lineagePropertyKey * @param importEdge import edge * @param vertexIdPair pair of import id and corresponding Gradoop vertex id * @return pair of import target vertex id and EPGM edge - * @throws Exception + * @throws Exception on failure */ @Override public Tuple2 join(ImportEdge importEdge, @@ -91,9 +91,6 @@ public Tuple2 join(ImportEdge importEdge, return reuseTuple; } - /** - * {@inheritDoc} - */ @Override public TypeInformation> getProducedType() { return new TupleTypeInfo<>(getKeyTypeInfo(), diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/InitVertex.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/InitVertex.java index 89ee8d8483a6..14f088441dcb 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/InitVertex.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/InitVertex.java @@ -72,7 +72,7 @@ public InitVertex(EPGMVertexFactory epgmVertexFactory, * * @param importVertex import vertex * @return triple containing import and EPGM id as well as the EPGM vertex - * @throws Exception + * @throws Exception on failure */ @Override public Tuple3 map(ImportVertex importVertex) throws @@ -88,9 +88,6 @@ public Tuple3 map(ImportVertex importVertex) throws return reuseTuple; } - /** - * {@inheritDoc} - */ @Override public TypeInformation> getProducedType() { return new TupleTypeInfo<>(getKeyTypeInfo(), diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/UpdateEdge.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/UpdateEdge.java index dbaeb50c1eaf..e3fb1b4a85cb 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/UpdateEdge.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/graph/functions/UpdateEdge.java @@ -38,7 +38,7 @@ public class UpdateEdge> * @param targetIdEdgePair import target id and EPGM edge * @param vertexIdPair import target vertex id and EPGM vertex id * @return EPGM edge with updated target vertex id - * @throws Exception + * @throws Exception on failure */ @Override public E join(Tuple2 targetIdEdgePair, diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/table/PrintTableSink.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/table/PrintTableSink.java new file mode 100644 index 000000000000..3e52c4a62a1c --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/table/PrintTableSink.java @@ -0,0 +1,100 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.io.impl.table; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.sinks.BatchTableSink; +import org.apache.flink.types.Row; + + +/** + * A batch table sink that prints the contents of specified fields of a table to the console. + */ +public class PrintTableSink implements BatchTableSink { + + /** + * Array containing the names of the fields of the table. + */ + private String[] fieldNames; + + /** + * Array containing the types of the fields of the table. + */ + private TypeInformation[] fieldTypes; + + /** + * Get the type of the table that is output through this sink. + * + * @return type of the table + */ + public TypeInformation getOutputType() { + return new RowTypeInfo(fieldTypes); + } + + /** + * Get the names of the fields of the table. + * + * @return array containing the field names + */ + public String[] getFieldNames() { + String[] copy = new String[fieldNames.length]; + System.arraycopy(fieldNames, 0, copy, 0, fieldNames.length); + return copy; + } + + /** + * Get the types of the fields of the table. + * + * @return array containing the field types + */ + public TypeInformation[] getFieldTypes() { + TypeInformation[] copy = new TypeInformation[fieldTypes.length]; + System.arraycopy(fieldTypes, 0, copy, 0, fieldTypes.length); + return copy; + } + + /** + * Configure the table sink with field names and types. + * + * @param fieldNames names of the fields of the table + * @param fieldTypes types of the fields of the table + * @return the configured table sink + */ + public PrintTableSink configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = new String[fieldNames.length]; + System.arraycopy(fieldNames, 0, this.fieldNames, 0, fieldNames.length); + this.fieldTypes = new TypeInformation[fieldTypes.length]; + System.arraycopy(fieldTypes, 0, this.fieldTypes, 0, fieldTypes.length); + return this; + } + + /** + * Emit a dataset (e.g. the one wrapped by a batch table) + * through this table sink by printing it to the console. + * + * @param dataSet the dataset to be printed + */ + @Override + public void emitDataSet(DataSet dataSet) { + try { + dataSet.print(); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/table/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/table/package-info.java new file mode 100644 index 000000000000..460fd4d82701 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/table/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains all classes related to the input and output of Flink tables. + */ +package org.gradoop.flink.io.impl.table; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/TLFDataSource.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/TLFDataSource.java index d73d787dfb7c..b79b14d2f6f9 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/TLFDataSource.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/TLFDataSource.java @@ -65,7 +65,7 @@ public TLFDataSource(String tlfPath, GradoopFlinkConfig config) { * @param config Gradoop Flink configuration */ public TLFDataSource(String tlfPath, String tlfVertexDictionaryPath, - String tlfEdgeDictionaryPath, GradoopFlinkConfig config) throws Exception { + String tlfEdgeDictionaryPath, GradoopFlinkConfig config) { super(tlfPath, tlfVertexDictionaryPath, tlfEdgeDictionaryPath, config); ExecutionEnvironment env = config.getExecutionEnvironment(); if (hasVertexDictionary()) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/Dictionary.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/Dictionary.java index 095390e462c2..5d4075f5b9f0 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/Dictionary.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/Dictionary.java @@ -34,7 +34,7 @@ public class Dictionary * * @param iterable containing tuples of integer and string * @param collector collects one map from integer to string - * @throws Exception + * @throws Exception on failure */ @Override public void reduce( diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/DictionaryEntry.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/DictionaryEntry.java index ad3cb762d20c..8b1c38519d5a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/DictionaryEntry.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/DictionaryEntry.java @@ -45,7 +45,7 @@ public DictionaryEntry() { * * @param tuple tuple received from TextInputFormat, for each line * @return tuple of the text, which was split into integer and string - * @throws Exception + * @throws Exception on failure */ @Override public Tuple2 map(Tuple2 tuple) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/EdgeLabelDecoder.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/EdgeLabelDecoder.java index 6ad351b9623a..7d4cd6e34311 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/EdgeLabelDecoder.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/EdgeLabelDecoder.java @@ -39,9 +39,6 @@ public class EdgeLabelDecoder extends */ private Map edgeDictionary; - /** - * {@inheritDoc} - */ @Override public void open(Configuration parameters) throws Exception { super.open(parameters); @@ -50,9 +47,6 @@ public void open(Configuration parameters) throws Exception { .get(0); } - /** - * {@inheritDoc} - */ @Override public GraphTransaction map(GraphTransaction graphTransaction) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/ElementLabelEncoder.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/ElementLabelEncoder.java index ca76ddd8b0eb..ff52e0c5b32b 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/ElementLabelEncoder.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/ElementLabelEncoder.java @@ -89,7 +89,7 @@ public void open(Configuration parameters) throws Exception { * @param graphTransaction graph transaction with label format: * 'dictionarylabel' * @return graph transaction with label format: 'simpleLabel' (Integer) - * @throws Exception + * @throws Exception on failure */ @Override public GraphTransaction map(GraphTransaction graphTransaction) diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/VertexLabelDecoder.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/VertexLabelDecoder.java index 9fdcb35f7db9..6486fc100675 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/VertexLabelDecoder.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/functions/VertexLabelDecoder.java @@ -38,9 +38,6 @@ public class VertexLabelDecoder extends */ private Map vertexDictionary; - /** - * {@inheritDoc} - */ @Override public void open(Configuration parameters) throws Exception { super.open(parameters); @@ -49,9 +46,6 @@ public void open(Configuration parameters) throws Exception { .get(0); } - /** - * {@inheritDoc} - */ @Override public GraphTransaction map(GraphTransaction graphTransaction) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/inputformats/TLFRecordReader.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/inputformats/TLFRecordReader.java index f7c22b06299a..21db65cc0ce6 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/inputformats/TLFRecordReader.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/tlf/inputformats/TLFRecordReader.java @@ -79,7 +79,7 @@ public class TLFRecordReader extends RecordReader { * * @param split the split of the file containing all TLF content * @param conf the configuration of the task attempt context - * @throws IOException + * @throws IOException on failure */ public TLFRecordReader(FileSplit split, Configuration conf) throws IOException { @@ -98,7 +98,7 @@ public TLFRecordReader(FileSplit split, Configuration conf) throws * @param key the new key * @param value the new value * @return true if a key/value pair was found - * @throws IOException + * @throws IOException on failure */ private boolean next(LongWritable key, Text value) throws IOException { if (fsin.getPos() < end && @@ -135,7 +135,7 @@ private boolean next(LongWritable key, Text value) throws IOException { * @param withinBlock specifies if match is within the graph block * @return true if match was found or the end of file was reached, so * that the current block can be closed - * @throws IOException + * @throws IOException on failure */ private boolean readUntilMatch(byte[] match, boolean withinBlock) throws IOException { @@ -169,7 +169,7 @@ private boolean readUntilMatch(byte[] match, boolean withinBlock) throws /** * Closes open buffers * - * @throws IOException + * @throws IOException on failure */ @Override public void close() throws IOException { @@ -181,7 +181,7 @@ public void close() throws IOException { * Returns the current process of input streaming. * * @return percentage of the completion - * @throws IOException + * @throws IOException on failure */ @Override public float getProgress() throws IOException { @@ -192,8 +192,8 @@ public float getProgress() throws IOException { * Returns the current key. * * @return the current key. - * @throws IOException - * @throws InterruptedException + * @throws IOException on failure + * @throws InterruptedException if interrupted */ @Override public LongWritable getCurrentKey() throws IOException, @@ -205,8 +205,8 @@ public LongWritable getCurrentKey() throws IOException, * Returns the current value. * * @return the current value - * @throws IOException - * @throws InterruptedException + * @throws IOException on failure + * @throws InterruptedException if interrupted */ @Override public Text getCurrentValue() throws IOException, InterruptedException { @@ -218,8 +218,8 @@ public Text getCurrentValue() throws IOException, InterruptedException { * * @param split the split of the file containing all TLF content * @param context current task attempt context - * @throws IOException - * @throws InterruptedException + * @throws IOException on failure + * @throws InterruptedException if interrupted */ @Override public void initialize(InputSplit split, TaskAttemptContext context) @@ -230,8 +230,8 @@ public void initialize(InputSplit split, TaskAttemptContext context) * Reads the next kex/value pair from the input for processing. * * @return true if a key/value pair was found - * @throws IOException - * @throws InterruptedException + * @throws IOException on failure + * @throws InterruptedException if interrupted */ @Override public boolean nextKeyValue() throws IOException, InterruptedException { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/api/epgm/LogicalGraphOperators.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/api/epgm/LogicalGraphOperators.java index 00ca5f9f120a..190b99721387 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/api/epgm/LogicalGraphOperators.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/api/epgm/LogicalGraphOperators.java @@ -17,6 +17,7 @@ import org.apache.flink.api.common.functions.FilterFunction; import org.apache.flink.api.java.DataSet; +import org.gradoop.common.model.impl.metadata.MetaData; import org.gradoop.common.model.impl.pojo.Edge; import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.common.model.impl.pojo.Vertex; @@ -30,6 +31,7 @@ import org.gradoop.flink.model.api.operators.UnaryGraphToCollectionOperator; import org.gradoop.flink.model.impl.epgm.GraphCollection; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.CAPFQueryResult; import org.gradoop.flink.model.impl.operators.grouping.Grouping; import org.gradoop.flink.model.impl.operators.grouping.GroupingStrategy; import org.gradoop.flink.model.impl.operators.matching.common.MatchStrategy; @@ -51,135 +53,37 @@ public interface LogicalGraphOperators extends GraphBaseOperators { //---------------------------------------------------------------------------- /** - * Evaluates the given query using the Cypher query engine. The engine uses default morphism - * strategies, which is vertex homomorphism and edge isomorphism. The vertex and edge data of - * the data graph elements is attached to the resulting vertices. - * - * Note, that this method used no statistics about the data graph which may result in bad - * runtime performance. Use {@link LogicalGraphOperators#cypher(String, GraphStatistics)} to - * provide statistics for the query planner. - * - * @param query Cypher query - * @return graph collection containing matching subgraphs - * @deprecated because of API restructuring. - * Please use {@link LogicalGraph#query(String)} instead. - */ - @Deprecated - GraphCollection cypher(String query); - - /** - * Evaluates the given query using the Cypher query engine. The engine uses default morphism - * strategies, which is vertex homomorphism and edge isomorphism. The vertex and edge data of - * the data graph elements is attached to the resulting vertices. - * - * Note, that this method used no statistics about the data graph which may result in bad - * runtime performance. Use {@link LogicalGraphOperators#cypher(String, GraphStatistics)} to - * provide statistics for the query planner. - * - * In addition, the operator can be supplied with a construction pattern allowing the creation - * of new graph elements based on variable bindings of the match pattern. Consider the following - * example: - * - *

-   * graph.cypher(
-   *  "MATCH (a:Author)-[:WROTE]->(:Paper)<-[:WROTE]-(b:Author) WHERE a <> b",
-   *  "(a)-[:CO_AUTHOR]->(b)")
-   * 
-   * 
- * - * The query pattern is looking for pairs of authors that worked on the same paper. The - * construction pattern defines a new edge of type CO_AUTHOR between the two entities. + * Evaluates the given cypher query using CAPF (Cypher for Apache Flink). CAPF implements the + * default cypher morphism strategies, which is vertex homomorphism and edge isomorphism. The + * result is a CAPFQueryResult, containing a flink table that can be converted to a + * GraphCollection, if it contains vertices or edges. * - * @param query Cypher query string - * @param constructionPattern Construction pattern - * @return graph collection containing the output of the construct pattern - * @deprecated because of API restructuring. - * Please use {@link LogicalGraph#query(String, String)} instead. + * @param query the query string + * @param metaData metaData object + * @return the result, containing a flink table and possibly a GraphCollection */ - @Deprecated - GraphCollection cypher(String query, String constructionPattern); + CAPFQueryResult cypher(String query, MetaData metaData) throws Exception; /** - * Evaluates the given query using the Cypher query engine. The engine uses default morphism - * strategies, which is vertex homomorphism and edge isomorphism. The vertex and edge data of - * the data graph elements is attached to the resulting vertices. + * Evaluates the given cypher query using CAPF (Cypher for Apache Flink). CAPF implements the + * default cypher morphism strategies, which is vertex homomorphism and edge isomorphism. The + * result is a CAPFQueryResult, containing a flink table that can be converted to a + * GraphCollection, if it contains vertices or edges. + *

+ * In this overloaded function, the property maps are constructed automatically. This is + * a lot slower and actually requires the job to be split in two parts to collect the property + * maps. * - * @param query Cypher query - * @param graphStatistics statistics about the data graph - * @return graph collection containing matching subgraphs - * @deprecated because of API restructuring. - * Please use {@link LogicalGraph#query(String, GraphStatistics)} instead. + * @param query the query string + * @return the result, containing a flink table and possibly a GraphCollection */ - @Deprecated - GraphCollection cypher(String query, GraphStatistics graphStatistics); + CAPFQueryResult cypher(String query) throws Exception; /** * Evaluates the given query using the Cypher query engine. The engine uses default morphism * strategies, which is vertex homomorphism and edge isomorphism. The vertex and edge data of * the data graph elements is attached to the resulting vertices. - * - * In addition, the operator can be supplied with a construction pattern allowing the creation - * of new graph elements based on variable bindings of the match pattern. Consider the following - * example: - * - *

-   * graph.cypher(
-   *  "MATCH (a:Author)-[:WROTE]->(:Paper)<-[:WROTE]-(b:Author) WHERE a <> b",
-   *  "(a)-[:CO_AUTHOR]->(b)")
-   * 
-   * 
- * - * The query pattern is looking for pairs of authors that worked on the same paper. The - * construction pattern defines a new edge of type CO_AUTHOR between the two entities. - * - * @param query Cypher query - * @param constructionPattern Construction pattern - * @param graphStatistics statistics about the data graph - * @return graph collection containing the output of the construct pattern - * @deprecated because of API restructuring. - * Please use {@link LogicalGraph#query(String, String, GraphStatistics)} instead. - */ - @Deprecated - GraphCollection cypher(String query, String constructionPattern, GraphStatistics graphStatistics); - - /** - * Evaluates the given query using the Cypher query engine. - * - * @param query Cypher query - * @param attachData attach original vertex and edge data to the result - * @param vertexStrategy morphism setting for vertex mapping - * @param edgeStrategy morphism setting for edge mapping - * @param graphStatistics statistics about the data graph - * @return graph collection containing matching subgraphs - * @deprecated because of API restructuring. - * Please use {@link LogicalGraph#query(String, boolean, MatchStrategy, MatchStrategy, GraphStatistics)} instead. - */ - @Deprecated - GraphCollection cypher(String query, boolean attachData, - MatchStrategy vertexStrategy, MatchStrategy edgeStrategy, GraphStatistics graphStatistics); - - /** - * Evaluates the given query using the Cypher query engine. - * - * @param query Cypher query - * @param constructionPattern Construction pattern - * @param attachData attach original vertex and edge data to the result - * @param vertexStrategy morphism setting for vertex mapping - * @param edgeStrategy morphism setting for edge mapping - * @param graphStatistics statistics about the data graph - * @return graph collection containing matching subgraphs - * @deprecated because of API restructuring. - * Please use {@link LogicalGraph#query(String, String, boolean, MatchStrategy, MatchStrategy, GraphStatistics)} instead. - */ - @Deprecated - GraphCollection cypher(String query, String constructionPattern, boolean attachData, - MatchStrategy vertexStrategy, MatchStrategy edgeStrategy, GraphStatistics graphStatistics); - - /** - * Evaluates the given query using the Cypher query engine. The engine uses default morphism - * strategies, which is vertex homomorphism and edge isomorphism. The vertex and edge data of - * the data graph elements is attached to the resulting vertices. - * + *

* Note, that this method used no statistics about the data graph which may result in bad * runtime performance. Use {@link LogicalGraphOperators#query(String, GraphStatistics)} to * provide statistics for the query planner. @@ -193,11 +97,11 @@ GraphCollection cypher(String query, String constructionPattern, boolean attachD * Evaluates the given query using the Cypher query engine. The engine uses default morphism * strategies, which is vertex homomorphism and edge isomorphism. The vertex and edge data of * the data graph elements is attached to the resulting vertices. - * + *

* Note, that this method used no statistics about the data graph which may result in bad * runtime performance. Use {@link LogicalGraphOperators#query(String, GraphStatistics)} to * provide statistics for the query planner. - * + *

* In addition, the operator can be supplied with a construction pattern allowing the creation * of new graph elements based on variable bindings of the match pattern. Consider the following * example: @@ -208,11 +112,11 @@ GraphCollection cypher(String query, String constructionPattern, boolean attachD * "(a)-[:CO_AUTHOR]->(b)") * * - * + *

* The query pattern is looking for pairs of authors that worked on the same paper. The * construction pattern defines a new edge of type CO_AUTHOR between the two entities. * - * @param query Cypher query string + * @param query Cypher query string * @param constructionPattern Construction pattern * @return graph collection containing the output of the construct pattern */ @@ -223,7 +127,7 @@ GraphCollection cypher(String query, String constructionPattern, boolean attachD * strategies, which is vertex homomorphism and edge isomorphism. The vertex and edge data of * the data graph elements is attached to the resulting vertices. * - * @param query Cypher query + * @param query Cypher query * @param graphStatistics statistics about the data graph * @return graph collection containing matching subgraphs */ @@ -233,7 +137,7 @@ GraphCollection cypher(String query, String constructionPattern, boolean attachD * Evaluates the given query using the Cypher query engine. The engine uses default morphism * strategies, which is vertex homomorphism and edge isomorphism. The vertex and edge data of * the data graph elements is attached to the resulting vertices. - * + *

* In addition, the operator can be supplied with a construction pattern allowing the creation * of new graph elements based on variable bindings of the match pattern. Consider the following * example: @@ -244,13 +148,13 @@ GraphCollection cypher(String query, String constructionPattern, boolean attachD * "(a)-[:CO_AUTHOR]->(b)") * * - * + *

* The query pattern is looking for pairs of authors that worked on the same paper. The * construction pattern defines a new edge of type CO_AUTHOR between the two entities. * - * @param query Cypher query + * @param query Cypher query * @param constructionPattern Construction pattern - * @param graphStatistics statistics about the data graph + * @param graphStatistics statistics about the data graph * @return graph collection containing the output of the construct pattern */ GraphCollection query(String query, String constructionPattern, GraphStatistics graphStatistics); @@ -258,33 +162,34 @@ GraphCollection cypher(String query, String constructionPattern, boolean attachD /** * Evaluates the given query using the Cypher query engine. * - * @param query Cypher query - * @param attachData attach original vertex and edge data to the result - * @param vertexStrategy morphism setting for vertex mapping - * @param edgeStrategy morphism setting for edge mapping + * @param query Cypher query + * @param attachData attach original vertex and edge data to the result + * @param vertexStrategy morphism setting for vertex mapping + * @param edgeStrategy morphism setting for edge mapping * @param graphStatistics statistics about the data graph * @return graph collection containing matching subgraphs */ GraphCollection query(String query, boolean attachData, MatchStrategy vertexStrategy, - MatchStrategy edgeStrategy, GraphStatistics graphStatistics); + MatchStrategy edgeStrategy, GraphStatistics graphStatistics); /** * Evaluates the given query using the Cypher query engine. * - * @param query Cypher query + * @param query Cypher query * @param constructionPattern Construction pattern - * @param attachData attach original vertex and edge data to the result - * @param vertexStrategy morphism setting for vertex mapping - * @param edgeStrategy morphism setting for edge mapping - * @param graphStatistics statistics about the data graph + * @param attachData attach original vertex and edge data to the result + * @param vertexStrategy morphism setting for vertex mapping + * @param edgeStrategy morphism setting for edge mapping + * @param graphStatistics statistics about the data graph * @return graph collection containing matching subgraphs */ GraphCollection query(String query, String constructionPattern, boolean attachData, - MatchStrategy vertexStrategy, MatchStrategy edgeStrategy, GraphStatistics graphStatistics); + MatchStrategy vertexStrategy, MatchStrategy edgeStrategy, + GraphStatistics graphStatistics); /** * Creates a copy of the logical graph. - * + *

* Note that this method creates new graph head, vertex and edge instances. * * @return projected logical graph @@ -355,14 +260,14 @@ LogicalGraph transformGraphHead( * Returns a subgraph of the logical graph which contains only those vertices * and edges that fulfil the given vertex and edge filter function * respectively. - * + *

* Note, that the operator does not verify the consistency of the resulting * graph. Use {#toGellyGraph().subgraph()} for that behaviour. * - * @param vertexFilterFunction vertex filter function - * @param edgeFilterFunction edge filter function - * @return logical graph which fulfils the given predicates and is a subgraph - * of that graph + * @param vertexFilterFunction vertex filter function + * @param edgeFilterFunction edge filter function + * @return logical graph which fulfils the given predicates and is a subgraph + * of that graph */ default LogicalGraph subgraph(FilterFunction vertexFilterFunction, FilterFunction edgeFilterFunction) { @@ -375,15 +280,15 @@ default LogicalGraph subgraph(FilterFunction vertexFilterFunction, * Returns a subgraph of the logical graph which contains only those vertices * and edges that fulfil the given vertex and edge filter function * respectively. - * + *

* Note, that the operator does not verify the consistency of the resulting * graph. Use {#toGellyGraph().subgraph()} for that behaviour. * - * @param vertexFilterFunction vertex filter function - * @param edgeFilterFunction edge filter function - * @param strategy execution strategy for the operator - * @return logical graph which fulfils the given predicates and is a subgraph - * of that graph + * @param vertexFilterFunction vertex filter function + * @param edgeFilterFunction edge filter function + * @param strategy execution strategy for the operator + * @return logical graph which fulfils the given predicates and is a subgraph + * of that graph */ LogicalGraph subgraph(FilterFunction vertexFilterFunction, FilterFunction edgeFilterFunction, Subgraph.Strategy strategy); @@ -410,15 +315,14 @@ LogicalGraph subgraph(FilterFunction vertexFilterFunction, /** * Creates a condensed version of the logical graph by grouping vertices based on the specified * property keys. - * + *

* Vertices are grouped by the given property keys. Edges are implicitly grouped along with their * incident vertices. - * + *

* Note: To group vertices by their type label, one needs to add the specific symbol * {@link Grouping#LABEL_SYMBOL} to the respective grouping keys. * * @param vertexGroupingKeys property keys to group vertices - * * @return summary graph * @see Grouping */ @@ -427,19 +331,18 @@ LogicalGraph subgraph(FilterFunction vertexFilterFunction, /** * Creates a condensed version of the logical graph by grouping vertices and edges based on given * property keys. - * + *

* Vertices are grouped by the given property keys. Edges are implicitly grouped along with their * incident vertices and explicitly by the specified edge grouping keys. - * + *

* One needs to at least specify a list of vertex grouping keys. Any other argument may be * {@code null}. - * + *

* Note: To group vertices/edges by their type label, one needs to add the specific symbol * {@link Grouping#LABEL_SYMBOL} to the respective grouping keys. * * @param vertexGroupingKeys property keys to group vertices - * @param edgeGroupingKeys property keys to group edges - * + * @param edgeGroupingKeys property keys to group edges * @return summary graph * @see Grouping */ @@ -448,24 +351,23 @@ LogicalGraph subgraph(FilterFunction vertexFilterFunction, /** * Creates a condensed version of the logical graph by grouping vertices and edges based on given * property keys. - * + *

* Vertices are grouped by the given property keys. Edges are implicitly grouped along with their * incident vertices and explicitly by the specified edge grouping keys. Furthermore, one can * specify sets of vertex and edge aggregate functions which are applied on vertices/edges * represented by the same super vertex/edge. - * + *

* One needs to at least specify a list of vertex grouping keys. Any other argument may be * {@code null}. - * + *

* Note: To group vertices/edges by their type label, one needs to add the specific symbol * {@link Grouping#LABEL_SYMBOL} to the respective grouping keys. * - * @param vertexGroupingKeys property keys to group vertices + * @param vertexGroupingKeys property keys to group vertices * @param vertexAggregateFunctions aggregate functions to apply on super vertices - * @param edgeGroupingKeys property keys to group edges - * @param edgeAggregateFunctions aggregate functions to apply on super edges - * @param groupingStrategy execution strategy for vertex grouping - * + * @param edgeGroupingKeys property keys to group edges + * @param edgeAggregateFunctions aggregate functions to apply on super edges + * @param groupingStrategy execution strategy for vertex grouping * @return summary graph * @see Grouping */ @@ -479,9 +381,8 @@ LogicalGraph groupBy( * the vertex is relevant get joined first and then grouped. The relevant edges are specified * using the direction which may direct to the vertex, or from the vertex or both. * - * @param function aggregate function + * @param function aggregate function * @param edgeDirection incoming, outgoing edges or both - * * @return logical graph where vertices store aggregated information about connected edges */ LogicalGraph reduceOnEdges( @@ -492,9 +393,8 @@ LogicalGraph reduceOnEdges( * of relevant edges get joined first and then grouped by the vertex. The relevant edges are * specified using the direction which may direct to the vertex, or from the vertex or both. * - * @param function aggregate function + * @param function aggregate function * @param edgeDirection incoming, outgoing edges or both - * * @return logical graph where vertices store aggregated information about connected vertices */ LogicalGraph reduceOnNeighbors( @@ -533,10 +433,10 @@ LogicalGraph reduceOnNeighbors( * grouping operations. For example, specifying the vertex grouping keys A, B and C leads to * three differently grouped graphs {A,B,C},{A,B},{A} within the resulting graph collection. * - * @param vertexGroupingKeys grouping keys to group vertices + * @param vertexGroupingKeys grouping keys to group vertices * @param vertexAggregateFunctions aggregate functions to apply on super vertices - * @param edgeGroupingKeys grouping keys to group edges - * @param edgeAggregateFunctions aggregate functions to apply on super edges + * @param edgeGroupingKeys grouping keys to group edges + * @param edgeAggregateFunctions aggregate functions to apply on super edges * @return graph collection containing all resulting graphs */ GraphCollection groupVerticesByRollUp( @@ -549,16 +449,26 @@ GraphCollection groupVerticesByRollUp( * grouping operations. For example, specifying the edge grouping keys A, B and C leads to * three differently grouped graphs {A,B,C},{A,B},{A} within the resulting graph collection. * - * @param vertexGroupingKeys grouping keys to group vertices + * @param vertexGroupingKeys grouping keys to group vertices * @param vertexAggregateFunctions aggregate functions to apply on super vertices - * @param edgeGroupingKeys grouping keys to group edges - * @param edgeAggregateFunctions aggregate functions to apply on super edges + * @param edgeGroupingKeys grouping keys to group edges + * @param edgeAggregateFunctions aggregate functions to apply on super edges * @return graph collection containing all resulting graphs */ GraphCollection groupEdgesByRollUp( List vertexGroupingKeys, List vertexAggregateFunctions, List edgeGroupingKeys, List edgeAggregateFunctions); + /** + * Verifies this graph, removing dangling edges, i.e. edges pointing to or from + * a vertex not contained in this graph.
+ * This operator can be applied after an operator that has not checked the graphs validity. + * The graph head of this logical graph remains unchanged. + * + * @return this graph with all dangling edges removed. + */ + LogicalGraph verify(); + //---------------------------------------------------------------------------- // Binary Operators //---------------------------------------------------------------------------- diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/api/functions/AggregateFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/api/functions/AggregateFunction.java index 6e1c29dfc2c5..c5fd0b17977a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/api/functions/AggregateFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/api/functions/AggregateFunction.java @@ -15,7 +15,7 @@ */ package org.gradoop.flink.model.api.functions; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.impl.operators.aggregation.Aggregation; @@ -49,7 +49,7 @@ public interface AggregateFunction extends Serializable { * @param element element used to get the increment * @return increment, may be NULL, which is handled in the operator */ - PropertyValue getIncrement(Element element); + PropertyValue getIncrement(EPGMElement element); /** * Returns whether this function aggregates vertices. diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/api/operators/Operator.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/api/operators/Operator.java index 8bdbe5c7ac81..e219fb6ffeda 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/api/operators/Operator.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/api/operators/Operator.java @@ -21,8 +21,11 @@ public interface Operator { /** * Returns the operators name. + * The operator name is the same as the class name, per default. * * @return operator name */ - String getName(); + default String getName() { + return this.getClass().getName(); + } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/epgm/GraphCollection.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/epgm/GraphCollection.java index 0116d6ca5156..ec57a1f346c4 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/epgm/GraphCollection.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/epgm/GraphCollection.java @@ -146,9 +146,7 @@ public DataSet getEdgesByLabel(String label) { return layout.getEdgesByLabel(label); } - /** - * {@inheritDoc} - */ + @Override public DataSet getGraphHeads() { return layout.getGraphHeads(); } @@ -167,9 +165,6 @@ public DataSet getGraphTransactions() { // Logical Graph / Graph Head Getters //---------------------------------------------------------------------------- - /** - * {@inheritDoc} - */ @Override public LogicalGraph getGraph(final GradoopId graphID) { // filter vertices and edges based on given graph id @@ -185,9 +180,6 @@ public LogicalGraph getGraph(final GradoopId graphID) { getConfig()); } - /** - * {@inheritDoc} - */ @Override public GraphCollection getGraphs(final GradoopId... identifiers) { @@ -200,16 +192,13 @@ public GraphCollection getGraphs(final GradoopId... identifiers) { return getGraphs(graphIds); } - /** - * {@inheritDoc} - */ @Override public GraphCollection getGraphs(final GradoopIdSet identifiers) { DataSet newGraphHeads = this.getGraphHeads() .filter(new FilterFunction() { @Override - public boolean filter(GraphHead graphHead) throws Exception { + public boolean filter(GraphHead graphHead) { return identifiers.contains(graphHead.getId()); } }); @@ -230,33 +219,21 @@ public boolean filter(GraphHead graphHead) throws Exception { // Unary Operators //---------------------------------------------------------------------------- - /** - * {@inheritDoc} - */ @Override public GraphCollection select(final FilterFunction predicate) { return callForCollection(new Selection(predicate)); } - /** - * {@inheritDoc} - */ @Override public GraphCollection sortBy(String propertyKey, Order order) { throw new NotImplementedException(); } - /** - * {@inheritDoc} - */ @Override public GraphCollection limit(int n) { return callForCollection(new Limit(n)); } - /** - * {@inheritDoc} - */ @Override public GraphCollection match( String pattern, @@ -272,25 +249,16 @@ public GraphCollection match( // Binary Operators //---------------------------------------------------------------------------- - /** - * {@inheritDoc} - */ @Override public GraphCollection union(GraphCollection otherCollection) { return callForCollection(new Union(), otherCollection); } - /** - * {@inheritDoc} - */ @Override public GraphCollection intersect(GraphCollection otherCollection) { return callForCollection(new Intersection(), otherCollection); } - /** - * {@inheritDoc} - */ @Override public GraphCollection intersectWithSmallResult( GraphCollection otherCollection) { @@ -298,17 +266,11 @@ public GraphCollection intersectWithSmallResult( otherCollection); } - /** - * {@inheritDoc} - */ @Override public GraphCollection difference(GraphCollection otherCollection) { return callForCollection(new Difference(), otherCollection); } - /** - * {@inheritDoc} - */ @Override public GraphCollection differenceWithSmallResult( GraphCollection otherCollection) { @@ -316,17 +278,11 @@ public GraphCollection differenceWithSmallResult( otherCollection); } - /** - * {@inheritDoc} - */ @Override public DataSet equalsByGraphIds(GraphCollection other) { return new CollectionEqualityByGraphIds().execute(this, other); } - /** - * {@inheritDoc} - */ @Override public DataSet equalsByGraphElementIds(GraphCollection other) { return new CollectionEquality( @@ -335,9 +291,6 @@ public DataSet equalsByGraphElementIds(GraphCollection other) { new EdgeToIdString(), true).execute(this, other); } - /** - * {@inheritDoc} - */ @Override public DataSet equalsByGraphElementData(GraphCollection other) { return new CollectionEquality( @@ -346,9 +299,6 @@ public DataSet equalsByGraphElementData(GraphCollection other) { new EdgeToDataString(), true).execute(this, other); } - /** - * {@inheritDoc} - */ @Override public DataSet equalsByGraphData(GraphCollection other) { return new CollectionEquality( @@ -361,18 +311,12 @@ public DataSet equalsByGraphData(GraphCollection other) { // Auxiliary Operators //---------------------------------------------------------------------------- - /** - * {@inheritDoc} - */ @Override public GraphCollection callForCollection( UnaryCollectionToCollectionOperator op) { return op.execute(this); } - /** - * {@inheritDoc} - */ @Override public GraphCollection callForCollection( BinaryCollectionToCollectionOperator op, @@ -380,25 +324,16 @@ public GraphCollection callForCollection( return op.execute(this, otherCollection); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph callForGraph(UnaryCollectionToGraphOperator op) { return op.execute(this); } - /** - * {@inheritDoc} - */ @Override public GraphCollection apply(ApplicableUnaryGraphToGraphOperator op) { return callForCollection(op); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph reduce(ReducibleBinaryGraphToGraphOperator op) { return callForGraph(op); @@ -418,9 +353,6 @@ public BaseGraphCollectionFactory getF return config.getGraphCollectionFactory(); } - /** - * {@inheritDoc} - */ @Override public DataSet isEmpty() { return getGraphHeads() @@ -431,17 +363,11 @@ public DataSet isEmpty() { .map(new Not()); } - /** - * {@inheritDoc} - */ @Override public GraphCollection distinctById() { return callForCollection(new DistinctById()); } - /** - * {@inheritDoc} - */ @Override public GraphCollection distinctByIsomorphism() { return callForCollection(new DistinctByIsomorphism()); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/epgm/LogicalGraph.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/epgm/LogicalGraph.java index 48bb47d6ddfd..100f56fb0eb3 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/epgm/LogicalGraph.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/epgm/LogicalGraph.java @@ -18,6 +18,7 @@ import com.google.common.collect.Lists; import org.apache.flink.api.common.functions.FilterFunction; import org.apache.flink.api.java.DataSet; +import org.gradoop.common.model.impl.metadata.MetaData; import org.gradoop.common.model.impl.pojo.Edge; import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.common.model.impl.pojo.Vertex; @@ -42,6 +43,8 @@ import org.gradoop.flink.model.impl.operators.aggregation.Aggregation; import org.gradoop.flink.model.impl.operators.cloning.Cloning; import org.gradoop.flink.model.impl.operators.combination.Combination; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.CAPFQuery; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.CAPFQueryResult; import org.gradoop.flink.model.impl.operators.equality.GraphEquality; import org.gradoop.flink.model.impl.operators.exclusion.Exclusion; import org.gradoop.flink.model.impl.operators.grouping.Grouping; @@ -65,6 +68,7 @@ import org.gradoop.flink.model.impl.operators.tostring.functions.VertexToDataString; import org.gradoop.flink.model.impl.operators.tostring.functions.VertexToIdString; import org.gradoop.flink.model.impl.operators.transformation.Transformation; +import org.gradoop.flink.model.impl.operators.verify.Verify; import org.gradoop.flink.util.GradoopFlinkConfig; import java.io.IOException; @@ -74,14 +78,14 @@ /** * A logical graph is one of the base concepts of the Extended Property Graph Model. A logical graph * encapsulates three concepts: - * + *

* - a so-called graph head, that stores information about the graph (i.e. label and properties) * - a set of vertices assigned to the graph * - a set of directed, possibly parallel edges assigned to the graph - * + *

* Furthermore, a logical graph provides operations that are performed on the underlying data. These * operations result in either another logical graph or in a {@link GraphCollection}. - * + *

* A logical graph is wrapping a {@link LogicalGraphLayout} which defines, how the graph is * represented in Apache Flink. Note that the LogicalGraph also implements that interface and * just forward the calls to the layout. This is just for convenience and API synchronicity. @@ -163,134 +167,63 @@ public DataSet getEdgesByLabel(String label) { // Unary Operators //---------------------------------------------------------------------------- - /** - * {@inheritDoc} - */ - @Override - @Deprecated - public GraphCollection cypher(String query) { - return cypher(query, new GraphStatistics(1, 1, 1, 1)); - } - - /** - * {@inheritDoc} - */ - @Override - @Deprecated - public GraphCollection cypher(String query, String constructionPattern) { - return cypher(query, constructionPattern, new GraphStatistics(1, 1, 1, 1)); - } - - /** - * {@inheritDoc} - */ - @Override - @Deprecated - public GraphCollection cypher(String query, GraphStatistics graphStatistics) { - return cypher(query, true, - MatchStrategy.HOMOMORPHISM, MatchStrategy.ISOMORPHISM, graphStatistics); - } - - /** - * {@inheritDoc} - */ - @Override - @Deprecated - public GraphCollection cypher(String query, String constructionPattern, - GraphStatistics graphStatistics) { - return cypher(query, constructionPattern, true, - MatchStrategy.HOMOMORPHISM, MatchStrategy.ISOMORPHISM, graphStatistics); - } - - - /** - * {@inheritDoc} - */ @Override - @Deprecated - public GraphCollection cypher(String query, boolean attachData, MatchStrategy vertexStrategy, - MatchStrategy edgeStrategy, GraphStatistics graphStatistics) { - return cypher(query, null, attachData, vertexStrategy, edgeStrategy, graphStatistics); + public CAPFQueryResult cypher(String query) throws Exception { + CAPFQuery capfQuery = new CAPFQuery( + query, this.config.getExecutionEnvironment() + ); + return capfQuery.execute(this); } - /** - * {@inheritDoc} - */ @Override - @Deprecated - public GraphCollection cypher(String query, String constructionPattern, boolean attachData, - MatchStrategy vertexStrategy, MatchStrategy edgeStrategy, GraphStatistics graphStatistics) { - return callForCollection(new CypherPatternMatching(query, constructionPattern, attachData, - vertexStrategy, edgeStrategy, graphStatistics)); + public CAPFQueryResult cypher(String query, MetaData metaData) throws Exception { + CAPFQuery capfQuery = new CAPFQuery( + query, metaData, this.config.getExecutionEnvironment()); + return capfQuery.execute(this); } - /** - * {@inheritDoc} - */ @Override public GraphCollection query(String query) { return query(query, new GraphStatistics(1, 1, 1, 1)); } - /** - * {@inheritDoc} - */ @Override public GraphCollection query(String query, String constructionPattern) { return query(query, constructionPattern, new GraphStatistics(1, 1, 1, 1)); } - /** - * {@inheritDoc} - */ @Override public GraphCollection query(String query, GraphStatistics graphStatistics) { return query(query, true, MatchStrategy.HOMOMORPHISM, MatchStrategy.ISOMORPHISM, graphStatistics); } - /** - * {@inheritDoc} - */ @Override public GraphCollection query(String query, String constructionPattern, - GraphStatistics graphStatistics) { + GraphStatistics graphStatistics) { return query(query, constructionPattern, true, MatchStrategy.HOMOMORPHISM, MatchStrategy.ISOMORPHISM, graphStatistics); } - - /** - * {@inheritDoc} - */ @Override public GraphCollection query(String query, boolean attachData, MatchStrategy vertexStrategy, - MatchStrategy edgeStrategy, GraphStatistics graphStatistics) { + MatchStrategy edgeStrategy, GraphStatistics graphStatistics) { return query(query, null, attachData, vertexStrategy, edgeStrategy, graphStatistics); } - /** - * {@inheritDoc} - */ @Override public GraphCollection query(String query, String constructionPattern, boolean attachData, - MatchStrategy vertexStrategy, MatchStrategy edgeStrategy, - GraphStatistics graphStatistics) { + MatchStrategy vertexStrategy, MatchStrategy edgeStrategy, + GraphStatistics graphStatistics) { return callForCollection(new CypherPatternMatching(query, constructionPattern, attachData, vertexStrategy, edgeStrategy, graphStatistics)); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph copy() { return callForGraph(new Cloning()); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph transform( TransformationFunction graphHeadTransformationFunction, @@ -320,9 +253,6 @@ public LogicalGraph transformEdges( return transform(null, null, edgeTransformationFunction); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph vertexInducedSubgraph( FilterFunction vertexFilterFunction) { @@ -331,9 +261,6 @@ public LogicalGraph vertexInducedSubgraph( new Subgraph<>(vertexFilterFunction, null, Subgraph.Strategy.VERTEX_INDUCED)); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph edgeInducedSubgraph( FilterFunction edgeFilterFunction) { @@ -341,51 +268,35 @@ public LogicalGraph edgeInducedSubgraph( return callForGraph(new Subgraph<>(null, edgeFilterFunction, Subgraph.Strategy.EDGE_INDUCED)); } - /** - * {@inheritDoc} - */ @Override - public LogicalGraph subgraph(FilterFunction vertexFilterFunction, + public LogicalGraph subgraph( + FilterFunction vertexFilterFunction, FilterFunction edgeFilterFunction, Subgraph.Strategy strategy) { + return callForGraph( new Subgraph<>(vertexFilterFunction, edgeFilterFunction, strategy)); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph aggregate(AggregateFunction... aggregateFunctions) { - return callForGraph(new Aggregation(aggregateFunctions)); + return callForGraph(new Aggregation<>(aggregateFunctions)); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph sample(SamplingAlgorithm algorithm) { return callForGraph(algorithm); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph groupBy(List vertexGroupingKeys) { return groupBy(vertexGroupingKeys, null); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph groupBy(List vertexGroupingKeys, List edgeGroupingKeys) { return groupBy(vertexGroupingKeys, null, edgeGroupingKeys, null, GroupingStrategy.GROUP_REDUCE); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph groupBy( List vertexGroupingKeys, List vertexAggregateFunctions, @@ -412,18 +323,12 @@ public LogicalGraph groupBy( return callForGraph(builder.build()); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph reduceOnEdges( EdgeAggregateFunction function, Neighborhood.EdgeDirection edgeDirection) { return callForGraph(new ReduceEdgeNeighborhood(function, edgeDirection)); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph reduceOnNeighbors( VertexAggregateFunction function, Neighborhood.EdgeDirection edgeDirection) { @@ -454,37 +359,30 @@ public GraphCollection groupEdgesByRollUp( edgeGroupingKeys, edgeAggregateFunctions)); } + @Override + public LogicalGraph verify() { + return callForGraph(new Verify<>()); + } + //---------------------------------------------------------------------------- // Binary Operators //---------------------------------------------------------------------------- - /** - * {@inheritDoc} - */ @Override public LogicalGraph combine(LogicalGraph otherGraph) { return callForGraph(new Combination(), otherGraph); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph overlap(LogicalGraph otherGraph) { return callForGraph(new Overlap(), otherGraph); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph exclude(LogicalGraph otherGraph) { return callForGraph(new Exclusion(), otherGraph); } - /** - * {@inheritDoc} - */ @Override public DataSet equalsByElementIds(LogicalGraph other) { return new GraphEquality( @@ -493,9 +391,6 @@ public DataSet equalsByElementIds(LogicalGraph other) { new EdgeToIdString(), true).execute(this, other); } - /** - * {@inheritDoc} - */ @Override public DataSet equalsByElementData(LogicalGraph other) { return new GraphEquality( @@ -504,9 +399,6 @@ public DataSet equalsByElementData(LogicalGraph other) { new EdgeToDataString(), true).execute(this, other); } - /** - * {@inheritDoc} - */ @Override public DataSet equalsByData(LogicalGraph other) { return new GraphEquality( @@ -519,42 +411,27 @@ public DataSet equalsByData(LogicalGraph other) { // Auxiliary Operators //---------------------------------------------------------------------------- - /** - * {@inheritDoc} - */ @Override public LogicalGraph callForGraph(UnaryBaseGraphToBaseGraphOperator operator) { return operator.execute(this); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph callForGraph(BinaryGraphToGraphOperator operator, LogicalGraph otherGraph) { return operator.execute(this, otherGraph); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph callForGraph(GraphsToGraphOperator operator, LogicalGraph... otherGraphs) { return operator.execute(this, otherGraphs); } - /** - * {@inheritDoc} - */ @Override public GraphCollection callForCollection(UnaryGraphToCollectionOperator operator) { return operator.execute(this); } - /** - * {@inheritDoc} - */ @Override public GraphCollection splitBy(String propertyKey) { return callForCollection(new Split(new PropertyGetter<>(Lists.newArrayList(propertyKey)))); @@ -564,9 +441,6 @@ public GraphCollection splitBy(String propertyKey) { // Utility methods //---------------------------------------------------------------------------- - /** - * {@inheritDoc} - */ @Override public DataSet isEmpty() { return getVertices() diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/EdgeFromIds.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/EdgeFromIds.java index 9a6e0f0833a1..0635d17b6bab 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/EdgeFromIds.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/EdgeFromIds.java @@ -63,7 +63,7 @@ public EdgeFromIds(EPGMEdgeFactory epgmEdgeFactory) { * @param idTriple triple containing (in that order) edge id, source vertex * id, target vertex id * @return EPGM edge - * @throws Exception + * @throws Exception on failure */ @Override public Edge map(Tuple3 idTriple) throws diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/ElementIdUpdater.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/ElementIdUpdater.java index 59a7c7492ee2..06a22544d70a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/ElementIdUpdater.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/ElementIdUpdater.java @@ -33,9 +33,6 @@ public class ElementIdUpdater implements MapFunction, EL> { - /** - * {@inheritDoc} - */ @Override public EL map(Tuple2 tuple2) { tuple2.f0.setId(tuple2.f1); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/InitGraphHead.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/InitGraphHead.java index 656cf287dc4c..b44f48bc1f97 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/InitGraphHead.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/InitGraphHead.java @@ -43,17 +43,11 @@ public InitGraphHead(EPGMGraphHeadFactory epgmGraphHeadFactory) { this.graphHeadFactory = epgmGraphHeadFactory; } - /** - * {@inheritDoc} - */ @Override public GraphHead map(Tuple1 idTuple) { return graphHeadFactory.initGraphHead(idTuple.f0); } - /** - * {@inheritDoc} - */ @Override public TypeInformation getProducedType() { return TypeExtractor.createTypeInfo(graphHeadFactory.getType()); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/Label.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/Label.java index b4972b043719..f1731c8bedde 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/Label.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/Label.java @@ -35,7 +35,7 @@ public String map(L l) throws Exception { } @Override - public String getKey(L l) throws Exception { + public String getKey(L l) { return l.getLabel(); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/functions/LabelIsIn.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/LabelIsIn.java similarity index 62% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/functions/LabelIsIn.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/LabelIsIn.java index 324ec6783264..197fef36b622 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/functions/LabelIsIn.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/LabelIsIn.java @@ -13,20 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.subgraph.functions; +package org.gradoop.flink.model.impl.functions.epgm; import com.google.common.collect.Sets; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.flink.model.impl.functions.filters.CombinableFilter; import java.util.Collection; +import java.util.HashSet; +import java.util.Objects; /** - * Filter function to check if an EPGM element's label is in a white list. + * Filter function to check if an EPGM elements label is in a white list. * - * @param element type + * @param The element type to filter. */ -public class LabelIsIn implements CombinableFilter { +public class LabelIsIn implements CombinableFilter { /** * White list of labels. @@ -34,13 +36,24 @@ public class LabelIsIn implements CombinableFilter { private final Collection labels; /** - * Constructor. + * Constructor for this filter using an array of labels. + * * @param labels white list of labels */ public LabelIsIn(String... labels) { this.labels = Sets.newHashSet(labels); } + /** + * Constructor for this filter using a collection of labels. + * + * @param labels A collection of accepted labels. + */ + public LabelIsIn(Collection labels) { + Objects.requireNonNull(labels); + this.labels = new HashSet<>(labels); + } + @Override public boolean filter(EL element) throws Exception { return labels.contains(element.getLabel()); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/PairElementWithPropertyValue.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/PairElementWithPropertyValue.java index f58b7f3c3566..a29795ef6baa 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/PairElementWithPropertyValue.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/PairElementWithPropertyValue.java @@ -51,9 +51,6 @@ public PairElementWithPropertyValue(String propertyKey) { this.reuseTuple = new Tuple2<>(); } - /** - * {@inheritDoc} - */ @Override public Tuple2 map(EL el) throws Exception { reuseTuple.f0 = el.getId(); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/Properties.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/Properties.java index 1c6dc7bde95b..a901c2e3cd15 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/Properties.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/Properties.java @@ -35,7 +35,7 @@ public org.gradoop.common.model.impl.properties.Properties map(L l) throws Excep } @Override - public org.gradoop.common.model.impl.properties.Properties getKey(L l) throws Exception { + public org.gradoop.common.model.impl.properties.Properties getKey(L l) { return l.getProperties(); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/VertexFromId.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/VertexFromId.java index 7477bf4b4e8f..5fac15be68d1 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/VertexFromId.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/epgm/VertexFromId.java @@ -52,7 +52,7 @@ public VertexFromId(EPGMVertexFactory vertexFactory) { * * @param gradoopId Gradoop identifier * @return EPGM vertex - * @throws Exception + * @throws Exception on failure */ @Override public Vertex map(Tuple1 gradoopId) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/graphcontainment/AddToGraph.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/graphcontainment/AddToGraph.java index 7bcc25f84668..a02cfcb2de4c 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/graphcontainment/AddToGraph.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/functions/graphcontainment/AddToGraph.java @@ -42,9 +42,6 @@ public AddToGraph(GraphHead graphHead) { this.graphHeadId = graphHead.getId(); } - /** - * {@inheritDoc} - */ @Override public GE map(GE graphElement) throws Exception { graphElement.addGraphId(graphHeadId); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/Aggregation.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/Aggregation.java index eb2db2550e03..db11ea731fa4 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/Aggregation.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/Aggregation.java @@ -16,13 +16,13 @@ package org.gradoop.flink.model.impl.operators.aggregation; import org.apache.flink.api.java.DataSet; -import org.gradoop.common.model.impl.pojo.Edge; -import org.gradoop.common.model.impl.pojo.GraphHead; -import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.api.entities.EPGMEdge; +import org.gradoop.common.model.api.entities.EPGMGraphHead; +import org.gradoop.common.model.api.entities.EPGMVertex; import org.gradoop.common.model.impl.properties.PropertyValue; -import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.api.epgm.BaseGraph; import org.gradoop.flink.model.api.functions.AggregateFunction; -import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.api.operators.UnaryBaseGraphToBaseGraphOperator; import org.gradoop.flink.model.impl.operators.aggregation.functions.AggregateElements; import org.gradoop.flink.model.impl.operators.aggregation.functions.CombinePartitionAggregates; import org.gradoop.flink.model.impl.operators.aggregation.functions.SetAggregateProperty; @@ -39,8 +39,17 @@ * Takes a logical graph and user defined aggregate functions as input. The * aggregate functions are applied on the logical graph and the resulting * aggregate is stored as additional properties at the result graph. + * + * @param The graph head type. + * @param The vertex type. + * @param The edge type. + * @param The type of the graph. */ -public class Aggregation implements UnaryGraphToGraphOperator { +public class Aggregation< + G extends EPGMGraphHead, + V extends EPGMVertex, + E extends EPGMEdge, + LG extends BaseGraph> implements UnaryBaseGraphToBaseGraphOperator { /** * User-defined aggregate functions which are applied on a single logical graph. @@ -61,20 +70,19 @@ public Aggregation(final AggregateFunction... aggregateFunctions) { } @Override - public LogicalGraph execute(LogicalGraph graph) { - DataSet vertices = graph.getVertices(); - DataSet edges = graph.getEdges(); + public LG execute(LG graph) { + DataSet vertices = graph.getVertices(); + DataSet edges = graph.getEdges(); DataSet> aggregate = aggregateVertices(vertices) .union(aggregateEdges(edges)) .reduceGroup(new CombinePartitionAggregates(aggregateFunctions)); - DataSet graphHead = graph.getGraphHead() - .map(new SetAggregateProperty(aggregateFunctions)) + DataSet graphHead = graph.getGraphHead() + .map(new SetAggregateProperty<>(aggregateFunctions)) .withBroadcastSet(aggregate, SetAggregateProperty.VALUE); - return graph.getConfig().getLogicalGraphFactory() - .fromDataSets(graphHead, vertices, edges); + return graph.getFactory().fromDataSets(graphHead, vertices, edges); } /** @@ -83,7 +91,7 @@ public LogicalGraph execute(LogicalGraph graph) { * @param vertices vertex data set * @return partition aggregate values mapped from their property key */ - private DataSet> aggregateVertices(DataSet vertices) { + private DataSet> aggregateVertices(DataSet vertices) { return vertices.combineGroup(new AggregateElements<>(aggregateFunctions.stream() .filter(AggregateFunction::isVertexAggregation) .collect(Collectors.toSet()))); @@ -95,14 +103,9 @@ private DataSet> aggregateVertices(DataSet ve * @param edges edge data set * @return partition aggregate values */ - private DataSet> aggregateEdges(DataSet edges) { + private DataSet> aggregateEdges(DataSet edges) { return edges.combineGroup(new AggregateElements<>(aggregateFunctions.stream() .filter(AggregateFunction::isEdgeAggregation) .collect(Collectors.toSet()))); } - - @Override - public String getName() { - return Aggregation.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/ApplyAggregation.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/ApplyAggregation.java index e4970563c3ee..3f7ffb688f27 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/ApplyAggregation.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/ApplyAggregation.java @@ -129,9 +129,4 @@ private DataSet>> aggregateEdges( .filter(AggregateFunction::isEdgeAggregation) .collect(Collectors.toSet()))); } - - @Override - public String getName() { - return ApplyAggregation.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/AggregateElements.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/AggregateElements.java index e169089c44b0..34de9fa6e9bb 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/AggregateElements.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/AggregateElements.java @@ -17,7 +17,7 @@ import org.apache.flink.api.common.functions.GroupCombineFunction; import org.apache.flink.util.Collector; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.api.functions.AggregateFunction; @@ -30,7 +30,7 @@ * * @param element type */ -public class AggregateElements +public class AggregateElements implements GroupCombineFunction> { /** diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/AggregateUtil.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/AggregateUtil.java index 91de01b7da55..1239fc8e8141 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/AggregateUtil.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/AggregateUtil.java @@ -15,7 +15,7 @@ */ package org.gradoop.flink.model.impl.operators.aggregation.functions; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.api.functions.AggregateDefaultValue; import org.gradoop.flink.model.api.functions.AggregateFunction; @@ -36,7 +36,8 @@ public class AggregateUtil { * @param aggregateFunctions aggregate functions * @return incremented aggregate map */ - static Map increment(Map aggregate, Element element, + static Map increment(Map aggregate, + EPGMElement element, Set aggregateFunctions) { for (AggregateFunction aggFunc : aggregateFunctions) { PropertyValue increment = aggFunc.getIncrement(element); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/SetAggregateProperty.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/SetAggregateProperty.java index c05c79983863..1c1ae72c84d1 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/SetAggregateProperty.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/SetAggregateProperty.java @@ -18,7 +18,7 @@ import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.api.java.functions.FunctionAnnotation; import org.apache.flink.configuration.Configuration; -import org.gradoop.common.model.impl.pojo.GraphHead; +import org.gradoop.common.model.api.entities.EPGMGraphHead; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.api.functions.AggregateFunction; @@ -30,10 +30,12 @@ /** * Sets aggregate values of a graph head. + * + * @param The graph head type. */ @FunctionAnnotation.ForwardedFields("id") -public class SetAggregateProperty - extends RichMapFunction { +public class SetAggregateProperty + extends RichMapFunction { /** * constant string for accessing broadcast variable "property values" @@ -84,7 +86,7 @@ public void open(Configuration parameters) throws Exception { } @Override - public GraphHead map(GraphHead graphHead) throws Exception { + public G map(G graphHead) throws Exception { aggregateValues.forEach(graphHead::setProperty); return graphHead; } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/containment/HasLabel.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/containment/HasLabel.java index 969be0507cdc..5638997d1a8f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/containment/HasLabel.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/containment/HasLabel.java @@ -15,7 +15,7 @@ */ package org.gradoop.flink.model.impl.operators.aggregation.functions.containment; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.api.functions.AggregateFunction; @@ -67,7 +67,7 @@ public HasLabel(String label, String aggregatePropertyKey) { } @Override - public PropertyValue getIncrement(Element element) { + public PropertyValue getIncrement(EPGMElement element) { return PropertyValue.create(element.getLabel().equals(label)); } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/count/Count.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/count/Count.java index 782f5de16736..438e4e5f65c4 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/count/Count.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/count/Count.java @@ -15,7 +15,7 @@ */ package org.gradoop.flink.model.impl.operators.aggregation.functions.count; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.api.functions.AggregateDefaultValue; import org.gradoop.flink.model.impl.operators.aggregation.functions.BaseAggregateFunction; @@ -43,7 +43,7 @@ public Count(String aggregatePropertyKey) { } @Override - public PropertyValue getIncrement(Element element) { + public PropertyValue getIncrement(EPGMElement element) { return PropertyValue.create(1L); } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/max/MaxProperty.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/max/MaxProperty.java index a9c42ba0bd24..e495f4a52364 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/max/MaxProperty.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/max/MaxProperty.java @@ -15,7 +15,7 @@ */ package org.gradoop.flink.model.impl.operators.aggregation.functions.max; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.impl.operators.aggregation.functions.BaseAggregateFunction; @@ -53,7 +53,7 @@ public MaxProperty(String propertyKey, String aggregatePropertyKey) { } @Override - public PropertyValue getIncrement(Element element) { + public PropertyValue getIncrement(EPGMElement element) { return element.getPropertyValue(propertyKey); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/min/MinProperty.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/min/MinProperty.java index d85fe07c35ec..9c7703984126 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/min/MinProperty.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/min/MinProperty.java @@ -15,7 +15,7 @@ */ package org.gradoop.flink.model.impl.operators.aggregation.functions.min; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.impl.operators.aggregation.functions.BaseAggregateFunction; @@ -53,7 +53,7 @@ public MinProperty(String propertyKey, String aggregatePropertyKey) { } @Override - public PropertyValue getIncrement(Element element) { + public PropertyValue getIncrement(EPGMElement element) { return element.getPropertyValue(propertyKey); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/sum/SumProperty.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/sum/SumProperty.java index 934eff622de5..e95a44828685 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/sum/SumProperty.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/aggregation/functions/sum/SumProperty.java @@ -15,7 +15,7 @@ */ package org.gradoop.flink.model.impl.operators.aggregation.functions.sum; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.impl.operators.aggregation.functions.BaseAggregateFunction; @@ -53,7 +53,7 @@ public SumProperty(String propertyKey, String aggregatePropertyKey) { } @Override - public PropertyValue getIncrement(Element element) { + public PropertyValue getIncrement(EPGMElement element) { return element.getPropertyValue(propertyKey); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/base/BinaryCollectionToCollectionOperatorBase.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/base/BinaryCollectionToCollectionOperatorBase.java index 3fce3eb91e04..bbad109958b4 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/base/BinaryCollectionToCollectionOperatorBase.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/base/BinaryCollectionToCollectionOperatorBase.java @@ -38,9 +38,6 @@ public abstract class BinaryCollectionToCollectionOperatorBase */ protected GraphCollection secondCollection; - /** - * {@inheritDoc} - */ @Override public GraphCollection execute( GraphCollection firstCollection, diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/Cloning.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/Cloning.java index d6c489487f2b..70ebfbda6535 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/Cloning.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/Cloning.java @@ -40,9 +40,6 @@ */ public class Cloning implements UnaryGraphToGraphOperator { - /** - * {@inheritDoc} - */ @Override public LogicalGraph execute(LogicalGraph graph) { @@ -92,12 +89,4 @@ public LogicalGraph execute(LogicalGraph graph) { return graph.getConfig().getLogicalGraphFactory() .fromDataSets(graphHead, vertices, edges); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return Cloning.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/EdgeSourceUpdateJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/EdgeSourceUpdateJoin.java index 857376c93831..6fa7b7084a6d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/EdgeSourceUpdateJoin.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/EdgeSourceUpdateJoin.java @@ -32,9 +32,7 @@ @FunctionAnnotation.ForwardedFieldsSecond("f1->sourceId") public class EdgeSourceUpdateJoin implements JoinFunction, E> { - /** - * {@inheritDoc} - */ + @Override public E join(E e, Tuple2 vertexTuple) { e.setSourceId(vertexTuple.f1); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/EdgeTargetUpdateJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/EdgeTargetUpdateJoin.java index ec3d58a7362d..11385c024862 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/EdgeTargetUpdateJoin.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/EdgeTargetUpdateJoin.java @@ -33,9 +33,6 @@ public class EdgeTargetUpdateJoin implements JoinFunction, E> { - /** - * {@inheritDoc} - */ @Override public E join(E e, Tuple2 vertexTuple) { e.setTargetId(vertexTuple.f1); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/ElementGraphUpdater.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/ElementGraphUpdater.java index 1ef8c43ad0bf..c1b8e3a7617f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/ElementGraphUpdater.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/ElementGraphUpdater.java @@ -49,9 +49,6 @@ public void open(Configuration parameters) throws Exception { .getBroadcastVariable(GRAPHID).get(0); } - /** - * {@inheritDoc} - */ @Override public EL map(EL element) { element.setGraphIds(GradoopIdSet.fromExisting(graphId)); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/Value0Of2ToId.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/Value0Of2ToId.java index 67a10ce7f019..bb77c1c10511 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/Value0Of2ToId.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cloning/functions/Value0Of2ToId.java @@ -37,9 +37,6 @@ public class Value0Of2ToId */ private final Tuple2 reuseTuple = new Tuple2<>(); - /** - * {@inheritDoc} - */ @Override public Tuple2 map(Tuple2 tuple2) { reuseTuple.setFields(tuple2.f0.getId(), tuple2.f1); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/combination/Combination.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/combination/Combination.java index 6ea4342e2176..c2023670f76d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/combination/Combination.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/combination/Combination.java @@ -51,12 +51,4 @@ public LogicalGraph execute(LogicalGraph firstGraph, return firstGraph.getConfig().getLogicalGraphFactory().fromDataSets(newVertexSet, newEdgeSet); } - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return Combination.class.getName(); - } - } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/combination/ReduceCombination.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/combination/ReduceCombination.java index 617949cbece9..8e3781ee4596 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/combination/ReduceCombination.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/combination/ReduceCombination.java @@ -36,9 +36,4 @@ public LogicalGraph execute(GraphCollection collection) { return collection.getConfig().getLogicalGraphFactory().fromDataSets( collection.getVertices(), collection.getEdges()); } - - @Override - public String getName() { - return ReduceCombination.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/package-info.java new file mode 100644 index 000000000000..6caafc48e5f6 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains implementations for the execution of cypher queries via CAPF and the handling of the + * resultes returned by CAPF + */ +package org.gradoop.flink.model.impl.operators.cypher.capf; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQuery.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQuery.java new file mode 100644 index 000000000000..5e9f60024603 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQuery.java @@ -0,0 +1,352 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple5; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.api.Table; +import org.apache.flink.types.Row; +import org.gradoop.common.model.impl.metadata.MetaData; +import org.gradoop.common.model.impl.metadata.PropertyMetaData; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.properties.Properties; +import org.gradoop.flink.io.impl.csv.metadata.CSVMetaDataSource; +import org.gradoop.flink.model.api.operators.Operator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.count.Count; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.EdgeLabelFilter; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.EdgeToTuple; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.IdOfF1; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.PropertyEncoder; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.ReplaceSourceId; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.ReplaceTargetId; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.TupleToRow; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.UniqueIdWithOffset; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.VertexLabelFilter; +import org.gradoop.flink.model.impl.operators.cypher.capf.query.functions.VertexToRow; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.CAPFQueryResult; +import org.opencypher.flink.api.CAPFSession; +import org.opencypher.flink.api.CAPFSession$; +import org.opencypher.flink.api.io.CAPFNodeTable; +import org.opencypher.flink.api.io.CAPFRelationshipTable; +import org.opencypher.flink.impl.table.FlinkCypherTable; +import org.opencypher.okapi.api.graph.PropertyGraph; +import org.opencypher.okapi.api.io.conversion.NodeMapping; +import org.opencypher.okapi.api.io.conversion.RelationshipMapping; +import org.opencypher.okapi.relational.api.io.EntityTable; +import scala.collection.JavaConversions; +import scala.collection.mutable.Seq; +import scala.reflect.ClassTag$; + +import java.util.ArrayList; +import java.util.List; + +import static org.gradoop.flink.model.impl.operators.cypher.capf.query.CAPFQueryConstants.EDGE_ID; +import static org.gradoop.flink.model.impl.operators.cypher.capf.query.CAPFQueryConstants.END_NODE; +import static org.gradoop.flink.model.impl.operators.cypher.capf.query.CAPFQueryConstants.NODE_ID; +import static org.gradoop.flink.model.impl.operators.cypher.capf.query.CAPFQueryConstants.OFFSET; +import static org.gradoop.flink.model.impl.operators.cypher.capf.query.CAPFQueryConstants.PROPERTY_PREFIX; +import static org.gradoop.flink.model.impl.operators.cypher.capf.query.CAPFQueryConstants.START_NODE; + + +/** + * Execute a cypher query on a LogicalGraph via the CAPF (Cypher for Apache Flink) + * API. + */ +public class CAPFQuery implements Operator { + + /** + * The query string. + */ + private String query; + + /** + * MetaData object + */ + private MetaData metaData; + + /** + * The CAPF session the query will be executed in. + */ + private CAPFSession session; + + /** + * The number of vertices by label, ordered alphabetically. + */ + private DataSet vertexCount; + + /** + * Mapping between the long ids and the original vertices. + */ + private DataSet> verticesWithIds; + + /** + * Mapping between the long ids and the original edges. + */ + private DataSet> edgesWithIds; + + /** + * Constructor + * + * @param query the query string + * @param env the execution environment + */ + public CAPFQuery(String query, ExecutionEnvironment env) { + + this.query = query; + + this.vertexCount = null; + this.session = CAPFSession$.MODULE$.create( + new org.apache.flink.api.scala.ExecutionEnvironment(env) + ); + } + + /** + * Constructor + * + * @param query the query string + * @param metaData metaData object + * @param env the execution environment + */ + public CAPFQuery( + String query, MetaData metaData, ExecutionEnvironment env) { + this.query = query; + this.metaData = metaData; + this.vertexCount = null; + this.session = CAPFSession$.MODULE$.create( + new org.apache.flink.api.scala.ExecutionEnvironment(env)); + } + + /** + * Execute a cypher query on a given graph via the CAPF API. + * + * @param graph the graph that the query shall be executed on + * @return the result of the query, either a graph collection or a flink table + * @throws Exception if the execution or IO fails. + */ + public CAPFQueryResult execute(LogicalGraph graph) throws Exception { + + if (metaData == null) { + graph = transformGraphProperties(graph); + metaData = new CSVMetaDataSource() + .fromTuples(new CSVMetaDataSource().tuplesFromGraph(graph).collect()); + } + // create flink tables of nodes as required by CAPF + List nodeTables = createNodeTables(graph); + + // create flink tables of relationships as required by CAPF + List relTables = createRelationshipTables(graph); + + // if there are no nodes, no edges can exit either, so we can terminate early + if (nodeTables.size() > 0) { + List> tables = new ArrayList<>( + nodeTables.subList(1, nodeTables.size())); + tables.addAll(relTables); + + Seq> tableSeq = + JavaConversions.asScalaBuffer(tables); + + PropertyGraph g = session.readFrom(nodeTables.get(0), tableSeq); + + // construct a CAPFQueryResult from the CAPFResult returned by CAPF + return new CAPFQueryResult( + g.cypher( + query, + g.cypher$default$2(), + g.cypher$default$3(), + g.cypher$default$4() + ), + verticesWithIds, + edgesWithIds, + graph.getConfig() + ); + } + + return null; + } + + /** + * Transform vertex and edge properties with types not yet supported by CAPF into string + * representations. + * + * @param graph the graph + * @return a graph with transformed vertex and edge properties + */ + private LogicalGraph transformGraphProperties(LogicalGraph graph) { + DataSet transformedVertices = graph.getVertices() + .map(new PropertyEncoder<>()); + DataSet transformedEdges = graph.getEdges() + .map(new PropertyEncoder<>()); + + return graph.getFactory().fromDataSets(transformedVertices, transformedEdges); + } + + /** + * Method to transform a DataSet of vertices into the flink table format + * required by CAPF: Unique long ids for each vertex, one table per + * vertex label and each property in a unique row field. + * + * @param graph the graph whose vertices should be transformed into CAPF tables + * @return a list of node tables, one table per vertex label + */ + private List createNodeTables(LogicalGraph graph) { + List nodeTables = new ArrayList<>(); + + verticesWithIds = graph.getVertices().map(new UniqueIdWithOffset<>()); + vertexCount = Count.count(graph.getVertices()); + + // construct a table for each vertex label + for (String label : metaData.getVertexLabels()) { + List propertyTypes = metaData.getVertexPropertyMetaData(label); + + // list of all row field types + TypeInformation[] types = new TypeInformation[propertyTypes.size() + 1]; + List propKeys = new ArrayList<>(propertyTypes.size()); + + // first field is long id + types[0] = TypeInformation.of(Long.class); + + for (int i = 0; i < propertyTypes.size(); i++) { + PropertyMetaData pmd = propertyTypes.get(i); + propKeys.add(pmd.getKey()); + types[i + 1] = TypeInformation.of(MetaData.getClassFromTypeString(pmd.getTypeString())); + } + + RowTypeInfo info = new RowTypeInfo(types); + + // zip all vertices of one label with a globally unique id + DataSet> verticesByLabelWithIds = + verticesWithIds.filter(new VertexLabelFilter(label)); + + // map vertices to row and wrap in scala DataSet + org.apache.flink.api.scala.DataSet scalaRowDataSet = + new org.apache.flink.api.scala.DataSet<>( + verticesByLabelWithIds.map(new VertexToRow(propKeys)).returns(info), + ClassTag$.MODULE$.apply(Row.class) + ); + + // build table schema string, naming each field in the table + StringBuilder schemaStringBuilder = new StringBuilder(NODE_ID); + NodeMapping nodeMapping = NodeMapping.withSourceIdKey(NODE_ID) + .withImpliedLabel(label); + + for (String propKey : propKeys) { + schemaStringBuilder.append(", ").append(PROPERTY_PREFIX).append(propKey); + + nodeMapping = nodeMapping.withPropertyKey(propKey, PROPERTY_PREFIX + propKey); + } + + String schemaString = schemaStringBuilder.toString(); + + // create table, add to node table list + Table vertexTable = session.tableEnv() + .fromDataSet(scalaRowDataSet).as(schemaString); + + nodeTables.add(CAPFNodeTable.fromMapping(nodeMapping, vertexTable)); + } + + return nodeTables; + } + + /** + * Method to transform a DataSet of edges into the flink table format + * required by CAPF: Unique long ids for each edge, source and target are long ids, + * one table per edge label and each property in a unique row field. + * + * @param graph the graph whose edges should be transformed into CAPF tables + * @return a list of edge tables, one table per edge label + */ + private List createRelationshipTables(LogicalGraph graph) { + List relTables = new ArrayList<>(); + + edgesWithIds = graph.getEdges().map(new UniqueIdWithOffset<>()) + .withBroadcastSet(vertexCount, OFFSET); + + // replace source and target with long ids + DataSet> edgeTuples = edgesWithIds + .map(new EdgeToTuple()) + .join(verticesWithIds) + .where(1).equalTo(new IdOfF1<>()).with(new ReplaceSourceId()) + .join(verticesWithIds) + .where(2).equalTo(new IdOfF1<>()).with(new ReplaceTargetId()); + + // construct a table for each edge label + for (String label : metaData.getEdgeLabels()) { + List propertyTypes = metaData.getEdgePropertyMetaData(label); + + // list of all row field types + TypeInformation[] types = new TypeInformation[propertyTypes.size() + 3]; + List propKeys = new ArrayList<>(propertyTypes.size()); + + // first fields are id, source id and target id + types[0] = TypeInformation.of(Long.class); // id + types[1] = TypeInformation.of(Long.class); // source + types[2] = TypeInformation.of(Long.class); // target + + // other fields are properties + for (int i = 0; i < propertyTypes.size(); i++) { + PropertyMetaData pmd = propertyTypes.get(i); + propKeys.add(pmd.getKey()); + types[i + 3] = TypeInformation.of(MetaData.getClassFromTypeString(pmd.getTypeString())); + } + + RowTypeInfo info = new RowTypeInfo(types); + + // zip all edges of one label with a globally unique id + DataSet> edgesByLabel = edgeTuples + .filter(new EdgeLabelFilter(label)); + + // map vertices to row and wrap in scala DataSet + org.apache.flink.api.scala.DataSet scalaRowDataSet = + new org.apache.flink.api.scala.DataSet<>( + edgesByLabel.map(new TupleToRow(propKeys)).returns(info), + ClassTag$.MODULE$.apply(Row.class) + ); + + // build table schema string, naming each field in the table + StringBuilder schemaStringBuilder = new StringBuilder(); + schemaStringBuilder + .append(EDGE_ID).append(", ") + .append(START_NODE).append(", ") + .append(END_NODE); + + RelationshipMapping relMapping = RelationshipMapping.withSourceIdKey(EDGE_ID) + .withSourceStartNodeKey(START_NODE) + .withSourceEndNodeKey(END_NODE) + .withRelType(label); + + for (String propKey : propKeys) { + schemaStringBuilder.append(", ").append(PROPERTY_PREFIX).append(propKey); + relMapping = relMapping.withPropertyKey(propKey, PROPERTY_PREFIX + propKey); + } + + String schemaString = schemaStringBuilder.toString(); + + // create table, add to relationship table list + Table edgeTable = session.tableEnv() + .fromDataSet(scalaRowDataSet).as(schemaString); + + relTables.add(CAPFRelationshipTable.fromMapping(relMapping, edgeTable)); + } + + return relTables; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQueryConstants.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQueryConstants.java new file mode 100644 index 000000000000..4c02a0b37d57 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQueryConstants.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query; + +/** + * Class containing string constants used for the execution of cypher queries via CAPF. + */ +class CAPFQueryConstants { + + /** + * Constant used to name the id field in a node table + */ + static final String NODE_ID = "node_id"; + + /** + * Constant used to name the id field in an edge table + */ + static final String EDGE_ID = "edge_id"; + + + /** + * Constant used to name the start node field in an edge table + */ + static final String START_NODE = "start_node"; + + + /** + * Constant used to name the end node field in an edge table + */ + static final String END_NODE = "end_node"; + + /** + * Constant used to mark a field as containing a property value + */ + static final String PROPERTY_PREFIX = "prop_"; + + + /** + * Constant used as broadcast name for the count offset + */ + static final String OFFSET = "offset"; + +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/EdgeLabelFilter.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/EdgeLabelFilter.java new file mode 100644 index 000000000000..88bbd11a6466 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/EdgeLabelFilter.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.FilterFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple5; +import org.gradoop.common.model.impl.properties.Properties; + +/** + * Method to filter edge tuples by their labels. + */ +@FunctionAnnotation.ReadFields("f3") +public class EdgeLabelFilter + implements FilterFunction> { + + /** + * The label to be filtered for. + */ + private String label; + + /** + * Constructor. + * + * @param label label to be filtered for + */ + public EdgeLabelFilter(String label) { + this.label = label; + } + + @Override + public boolean filter(Tuple5 tuple) throws Exception { + return tuple.f3.equals(label); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/EdgeToTuple.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/EdgeToTuple.java new file mode 100644 index 000000000000..d4ca5d43f161 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/EdgeToTuple.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple5; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.properties.Properties; + +/** + * MapFunction that maps tuples containing a Long id and an edge to an edge tuple. + */ + +@FunctionAnnotation.ForwardedFields( + "f0;" + + "f1.sourceId->f1;" + + "f1.targetId->f2;" + + "f1.label->f3;" + + "f1.properties->f4") +public class EdgeToTuple + implements MapFunction, + Tuple5> { + + /** + * Reduce object instantiations + */ + private Tuple5 returnTuple = new Tuple5<>(); + + @Override + public Tuple5 map( + Tuple2 tuple) throws Exception { + + Edge edge = tuple.f1; + + returnTuple.f0 = tuple.f0; + returnTuple.f1 = edge.getSourceId(); + returnTuple.f2 = edge.getTargetId(); + returnTuple.f3 = edge.getLabel(); + returnTuple.f4 = edge.getProperties(); + + return returnTuple; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/IdOfF1.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/IdOfF1.java new file mode 100644 index 000000000000..8abb10c306b1 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/IdOfF1.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Element; + +/** + * KeySelector that returns the id of a {@link Element} in the second field of a tuple. + * + * @param any GraphElement + */ +@FunctionAnnotation.ForwardedFields("f1.id->*") +public class IdOfF1 implements KeySelector, GradoopId> { + + @Override + public GradoopId getKey(Tuple2 tuple) { + return tuple.f1.getId(); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/PropertyEncoder.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/PropertyEncoder.java new file mode 100644 index 000000000000..066b9e4857ce --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/PropertyEncoder.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.MapFunction; +import org.gradoop.common.model.api.entities.EPGMElement; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.properties.Property; + +import java.math.BigDecimal; +import java.nio.charset.Charset; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Encodes property types not yet supported by CAPF to a String representation. + * @param an EPGMElement type + */ +public class PropertyEncoder implements MapFunction { + + + @Override + public E map(E e) throws Exception { + if (e.getProperties() != null) { + Class[] classes = {BigDecimal.class, GradoopId.class, Map.class, List.class, LocalDate.class, + LocalTime.class, LocalDateTime.class, Set.class}; + Set classSet = new HashSet<>(Arrays.asList(classes)); + for (Property prop : e.getProperties()) { + if (classSet.contains(prop.getValue().getType())) { + e.getProperties().set(prop.getKey(), + "CAPFProperty" + new String(prop.getValue().getRawBytes(), Charset.forName("UTF-8"))); + } + } + } + return e; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/ReplaceSourceId.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/ReplaceSourceId.java new file mode 100644 index 000000000000..ab48ef5139e9 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/ReplaceSourceId.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple5; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.properties.Properties; + +/** + * Replace the source GradoopId of an edge tuple with the matching Long id; + */ + +@FunctionAnnotation.ForwardedFieldsFirst("f0->f0;f2->f2;f3->f3") +@FunctionAnnotation.ForwardedFieldsSecond("f0->f1") +public class ReplaceSourceId implements JoinFunction< + Tuple5, + Tuple2, + Tuple5> { + + /** + * Reduce object instantiations + */ + private Tuple5 returnTuple = new Tuple5<>(); + + @Override + public Tuple5 join( + Tuple5 inputTuple, + Tuple2 vertexTuple + ) throws Exception { + returnTuple.f0 = inputTuple.f0; + returnTuple.f1 = vertexTuple.f0; + returnTuple.f2 = inputTuple.f2; + returnTuple.f3 = inputTuple.f3; + returnTuple.f4 = inputTuple.f4; + + return returnTuple; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/ReplaceTargetId.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/ReplaceTargetId.java new file mode 100644 index 000000000000..69bbe1715630 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/ReplaceTargetId.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple5; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.properties.Properties; + +/** + * Replace the source GradoopId of an edge tuple with the matching Long id; + */ + +@FunctionAnnotation.ForwardedFieldsFirst("f0->f0;f1->f1;f3->f3;f4->f4") +@FunctionAnnotation.ForwardedFieldsSecond("f0->f2") +public class ReplaceTargetId implements JoinFunction< + Tuple5, + Tuple2, + Tuple5> { + + /** + * Reduce object instantiations + */ + private Tuple5 returnTuple = new Tuple5<>(); + + @Override + public Tuple5 join( + Tuple5 inputTuple, + Tuple2 vertexTuple + ) throws Exception { + returnTuple.f0 = inputTuple.f0; + returnTuple.f1 = inputTuple.f1; + returnTuple.f2 = vertexTuple.f0; + returnTuple.f3 = inputTuple.f3; + returnTuple.f4 = inputTuple.f4; + return returnTuple; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/TupleToRow.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/TupleToRow.java new file mode 100644 index 000000000000..a45db98f5857 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/TupleToRow.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.RichMapFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple5; +import org.apache.flink.types.Row; +import org.gradoop.common.model.impl.properties.Properties; + +import java.util.List; + +/** + * RichMapFunction that maps edge tuples to rows. + * A new row field is created for each property that edges + * with this label can have. + */ + +@FunctionAnnotation.ForwardedFields("f0->f0;f1->f1;f2->f2") +public class TupleToRow extends RichMapFunction, Row> { + + /** + * List of all property keys the edges with this label have. + */ + private List propertyKeys; + + /** + * Reduce object instantiations + */ + private Row returnRow; + + /** + * Constructor + * + * @param propertyKeys list of all property keys edges with this label have + */ + public TupleToRow(List propertyKeys) { + this.propertyKeys = propertyKeys; + returnRow = new Row(propertyKeys.size() + 3); + } + + @Override + public Row map(Tuple5 tuple) throws Exception { + + returnRow.setField(0, tuple.f0); + returnRow.setField(1, tuple.f1); + returnRow.setField(2, tuple.f2); + + int i = 3; + for (String key : propertyKeys) { + if (tuple.f4.containsKey(key)) { + returnRow.setField(i, tuple.f4.get(key).getObject()); + } + i++; + } + return returnRow; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/UniqueIdWithOffset.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/UniqueIdWithOffset.java new file mode 100644 index 000000000000..3c4cc6c7316f --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/UniqueIdWithOffset.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.RichMapFunction; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.Configuration; + +import java.util.List; + +/** + * Zip each element in the dataset with a unique Long id. Its possible to set an offset via a + * broadcast set. + * + * @param the type of elements in the dataset + */ + +public class UniqueIdWithOffset extends RichMapFunction> { + + /** + * The parallelism this MapFunction is run at. + */ + private Long parallelism; + + /** + * A counter to make sure each id is only given once. + */ + private Long idCounter; + + /** + * Reduce object instantiations + */ + private Tuple2 returnTuple = new Tuple2<>(); + + @Override + public void open(Configuration parameters) { + + idCounter = 0L; + + RuntimeContext ctx = getRuntimeContext(); + // if an offset has been set, add it to the idCounter + if (ctx.hasBroadcastVariable("offset")) { + List offset = ctx.getBroadcastVariable("offset"); + if (offset.size() > 0) { + idCounter += offset.get(0); + } + } + parallelism = (long) ctx.getNumberOfParallelSubtasks(); + idCounter += (long) ctx.getIndexOfThisSubtask(); + } + + @Override + public Tuple2 map(E e) throws Exception { + returnTuple.f0 = idCounter; + returnTuple.f1 = e; + idCounter += parallelism; + return returnTuple; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/VertexLabelFilter.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/VertexLabelFilter.java new file mode 100644 index 000000000000..ce31703962fd --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/VertexLabelFilter.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.FilterFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.pojo.Vertex; + +/** + * Function filters tuples by the label of the {@link Vertex} in the second field. + */ +@FunctionAnnotation.ReadFields("f1") +public class VertexLabelFilter implements FilterFunction> { + + /** + * The label to be filtered for. + */ + private String label; + + /** + * Constructor. + * + * @param label label to be filtered for + */ + public VertexLabelFilter(String label) { + this.label = label; + } + + @Override + public boolean filter(Tuple2 tuple) throws Exception { + return tuple.f1.getLabel().equals(label); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/VertexToRow.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/VertexToRow.java new file mode 100644 index 000000000000..40331274da36 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/VertexToRow.java @@ -0,0 +1,69 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; + +import org.apache.flink.api.common.functions.RichMapFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; +import org.gradoop.common.model.impl.pojo.Vertex; + +import java.util.Collection; + +/** + * RichMapFunction that maps tuples containing a Long id and an vertex to row. + * A new row field is created for each property that vertices + * with this label can have. + */ + +@FunctionAnnotation.ForwardedFields("f0->f0") +@FunctionAnnotation.ReadFields("f1.properties") +public class VertexToRow extends RichMapFunction, Row> { + + /** + * List of all property keys the edges with this label have. + */ + private Collection propertyKeys; + + /** + * Reduce object instantiations + */ + private Row returnRow; + + /** + * Constructor + * + * @param propertyKeys list of all property keys edges with this label have + */ + public VertexToRow(Collection propertyKeys) { + this.propertyKeys = propertyKeys; + returnRow = new Row(propertyKeys.size() + 1); + } + + @Override + public Row map(Tuple2 tuple) throws Exception { + returnRow.setField(0, tuple.f0); + + int i = 1; + for (String key : propertyKeys) { + if (tuple.f1.hasProperty(key)) { + returnRow.setField(i, tuple.f1.getPropertyValue(key).getObject()); + } + i++; + } + return returnRow; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/package-info.java new file mode 100644 index 000000000000..bd9d0704168f --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/functions/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Functions required for the transformation of graphs to CAPF compatible Flink tables + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query.functions; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/package-info.java new file mode 100644 index 000000000000..dc58365577f7 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains implementation for the transformation of graphs to CAPF compatible Flink tables + * and the execution of cypher queries on them via CAPF + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/CAPFQueryResult.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/CAPFQueryResult.java new file mode 100644 index 000000000000..f03839a717da --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/CAPFQueryResult.java @@ -0,0 +1,282 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.result; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.tools.RuleSets; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.calcite.CalciteConfig; +import org.apache.flink.table.calcite.CalciteConfigBuilder; +import org.apache.flink.types.Row; +import org.gradoop.common.model.impl.id.GradoopIdSet; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.GraphHead; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.flink.model.impl.epgm.GraphCollection; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.functions.AddGradoopIdToRow; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.functions.AddNewGraphs; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.functions.AggregateGraphs; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.functions.CreateGraphHeadWithProperties; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.functions.PropertyDecoder; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.functions.SplitRow; +import org.gradoop.flink.util.GradoopFlinkConfig; +import org.opencypher.flink.api.CAPFSession; +import org.opencypher.flink.impl.CAPFRecords; +import org.opencypher.okapi.api.graph.CypherResult; +import org.opencypher.okapi.ir.api.expr.Expr; +import org.opencypher.okapi.ir.api.expr.Var; +import scala.collection.Iterator; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Wrapper containing the results of a CAPF query. + */ +public class CAPFQueryResult { + + /** + * Logical optimizer rules to be removed for better optimizer performance. + * + * {@code ProjectMergeRule:force_mode} would result in long optimization times. + */ + private static final List DISABLED_RULES = + Collections.singletonList("ProjectMergeRule:force_mode"); + + /** + * The wrapped CAPFRecords. + */ + private CAPFRecords records; + + /** + * Flag, set true iff the CAPFResult contains graph entities and can be transformed into a graph. + */ + private boolean isGraph; + + /** + * The CAPFSession that was used to create the result. + */ + private CAPFSession session; + + /** + * Mapping between the long ids and the original vertices. + */ + private DataSet> verticesWithIds; + + /** + * Mapping between the long ids and the original edges. + */ + private DataSet> edgesWithIds; + + /** + * The GradoopFlinkConfig. + */ + private GradoopFlinkConfig config; + + /** + * Constructor; + * + * @param result result of CAPF query + * @param verticesWithIds map between long id and original vertex + * @param edgesWithIds map between long id and original edge + * @param config the gradoop config + */ + public CAPFQueryResult( + CypherResult result, + DataSet> verticesWithIds, + DataSet> edgesWithIds, + GradoopFlinkConfig config) { + this.records = (CAPFRecords) result.records(); + this.verticesWithIds = verticesWithIds; + this.edgesWithIds = edgesWithIds; + this.config = config; + + this.session = ((CAPFRecords) result.records()).capf(); + this.isGraph = !records.header().entityVars().isEmpty(); + } + + + /** + * Returns true iff the result contained entities that can be returned as graph collection. + * + * @return true iff results contain graphs + */ + public boolean containsGraphs() { + return isGraph; + } + + /** + * Get the graphs contained in the CAPF query result. + * Returns null if the result contains no graphs. + * + * @return graphs contained in CAPF query iff there are any, else null + */ + public GraphCollection getGraphs() { + + if (!isGraph) { + return null; + } + + Set nodeVars = new HashSet<>(); + Set relVars = new HashSet<>(); + Set otherVars = new HashSet<>(); + + Iterator varIt = records.header().vars().iterator(); + while (varIt.hasNext()) { + otherVars.add(varIt.next()); + } + + Iterator nodeVarIt = records.header().nodeEntities().iterator(); + while (nodeVarIt.hasNext()) { + Var nodeVar = nodeVarIt.next(); + nodeVars.add(nodeVar); + otherVars.remove(nodeVar); + } + + Iterator relVarIt = records.header().relationshipEntities().iterator(); + while (relVarIt.hasNext()) { + Var relVar = relVarIt.next(); + relVars.add(relVar); + otherVars.remove(relVar); + } + + StringBuilder entityFieldsBuilder = new StringBuilder(); + for (Var var : nodeVars) { + entityFieldsBuilder.append(records.header().column((Expr) var)).append(","); + } + + for (Var var : relVars) { + entityFieldsBuilder.append(records.header().column((Expr) var)).append(","); + } + + StringBuilder otherFieldsBuilder = new StringBuilder(); + List otherVarNames = new ArrayList<>(); + + for (Var var : otherVars) { + otherVarNames.add(var.name()); + otherFieldsBuilder.append( + records.header().getColumn((Expr) var).get()).append(", "); + } + + String fieldString = entityFieldsBuilder.toString() + otherFieldsBuilder.toString(); + if (fieldString.length() > 0) { + fieldString = fieldString.substring(0, fieldString.length() - 1); + } + + TypeInformation rowTypeInfo = TypeInformation.of(Row.class); + + // Workaround for usable optimizer times + removeSlowOptimizationRule(); + + // entities, others, id + org.apache.flink.api.scala.DataSet scalarowsWithNewIds = session.tableEnv() + .toDataSet(records.table().table().select(fieldString), rowTypeInfo); + + DataSet rowsWithNewIds = scalarowsWithNewIds.javaSet() + .map(new AddGradoopIdToRow()); + + int entityFieldsCount = nodeVars.size() + relVars.size(); + int otherFieldsCount = otherVars.size(); + + DataSet graphHeads = rowsWithNewIds + .map(new CreateGraphHeadWithProperties( + entityFieldsCount, + entityFieldsCount + otherFieldsCount, + config.getGraphHeadFactory(), + otherVarNames) + ); + + DataSet> rowsWithGraphIdSets = rowsWithNewIds + .flatMap(new SplitRow(0, entityFieldsCount)) + .groupBy(0) + .reduceGroup(new AggregateGraphs<>()); + + DataSet vertices = + rowsWithGraphIdSets + .join(verticesWithIds) + .where(0) + .equalTo(0) + .with(new AddNewGraphs<>()); + + DataSet edges = + rowsWithGraphIdSets + .join(edgesWithIds) + .where(0) + .equalTo(0) + .with(new AddNewGraphs<>()); + + vertices = vertices.map(new PropertyDecoder<>()); + edges = edges.map(new PropertyDecoder<>()); + + return config.getGraphCollectionFactory().fromDataSets(graphHeads, vertices, edges); + } + + /** + * Workaround to remove slow logical optimization rules listed in + * {@link CAPFQueryResult#DISABLED_RULES}. + * + * This method accesses protected scala flink functions. See Issue #1221 + * (https://github.com/dbs-leipzig/gradoop/issues/1221). + */ + private void removeSlowOptimizationRule() { + List ruleList = new ArrayList<>(); + for (RelOptRule rule : session.tableEnv().getLogicalOptRuleSet()) { + if (!DISABLED_RULES.contains(rule.toString())) { + ruleList.add(rule); + } + } + CalciteConfigBuilder builder = new CalciteConfigBuilder() + .replaceLogicalOptRuleSet(RuleSets.ofList(ruleList)); + + // rebuild old calcite config + CalciteConfig calciteConfig = session.tableEnv().config().getCalciteConfig(); + if (calciteConfig.replacesDecoRuleSet()) { + builder.replaceDecoRuleSet(calciteConfig.getDecoRuleSet().get()); + } + if (calciteConfig.replacesNormRuleSet()) { + builder.replaceNormRuleSet(calciteConfig.getNormRuleSet().get()); + } + if (calciteConfig.replacesPhysicalOptRuleSet()) { + builder.replacePhysicalOptRuleSet(calciteConfig.getPhysicalOptRuleSet().get()); + } + if (calciteConfig.replacesSqlOperatorTable()) { + builder.replaceSqlOperatorTable(calciteConfig.getSqlOperatorTable().get()); + } + if (calciteConfig.getSqlParserConfig().isDefined()) { + builder.replaceSqlParserConfig(calciteConfig.getSqlParserConfig().get()); + } + if (calciteConfig.getSqlToRelConverterConfig().isDefined()) { + builder.replaceSqlToRelConverterConfig(calciteConfig.getSqlToRelConverterConfig().get()); + } + + session.tableEnv().config().setCalciteConfig(builder.build()); + } + + /** + * Get the flink table from the CAPF query result. + * + * @return flink table containing the CAPF query result + */ + public Table getTable() { + return records.table().table(); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexWithId.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AddGradoopIdToRow.java similarity index 54% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexWithId.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AddGradoopIdToRow.java index 77e67b0430dc..b12d3bb83d9c 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexWithId.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AddGradoopIdToRow.java @@ -13,36 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.functions; +package org.gradoop.flink.model.impl.operators.cypher.capf.result.functions; import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; import org.gradoop.common.model.impl.id.GradoopId; -import org.gradoop.common.model.impl.pojo.Vertex; /** - * Map a vertex to a tuple of the vertex and its id + * Extracts the first field of a Row, which is expected to be a GraphHead. */ -public class VertexWithId implements MapFunction> { - /** - * Reduce object instantiations - */ - private Tuple2 reuse; +public class AddGradoopIdToRow implements MapFunction { - /** - * Constructor - */ - public VertexWithId() { - this.reuse = new Tuple2<>(); - } - - /** - * {@inheritDoc} - */ @Override - public Tuple2 map(Vertex vertex) throws Exception { - reuse.f0 = vertex; - reuse.f1 = vertex.getId(); - return reuse; + public Row map(Row row) { + if (row.getArity() == 0) { + return null; + } else { + Row newRow = new Row(row.getArity() + 1); + int i = 0; + for (; i < row.getArity(); i++) { + newRow.setField(i, row.getField(i)); + } + newRow.setField(i, GradoopId.get()); + + return newRow; + } } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AddNewGraphs.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AddNewGraphs.java new file mode 100644 index 000000000000..028111583e68 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AddNewGraphs.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.result.functions; + +import org.apache.flink.api.common.functions.JoinFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.id.GradoopIdSet; +import org.gradoop.common.model.impl.pojo.GraphElement; + +/** + * Adds GradoopIds to the graph ids of a GraphElement based on a long id + * that is defined on both datasets. + * + * @param a type extending GraphElement, e.g. Vertex or Edge + */ + +@FunctionAnnotation.ReadFieldsFirst("f1") +@FunctionAnnotation.ReadFieldsSecond("f1") +public class AddNewGraphs + implements JoinFunction, Tuple2, E> { + + @Override + public E join( + Tuple2 idTuple, + Tuple2 elementTuple) throws Exception { + + E element = elementTuple.f1; + for (GradoopId id : idTuple.f1) { + element.addGraphId(id); + } + return element; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AggregateGraphs.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AggregateGraphs.java new file mode 100644 index 000000000000..6a07424174b6 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/AggregateGraphs.java @@ -0,0 +1,63 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.result.functions; + +import org.apache.flink.api.common.functions.GroupReduceFunction; +import org.apache.flink.api.java.functions.FunctionAnnotation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.id.GradoopIdSet; + +import java.util.Iterator; + +/** + * Aggregate function that aggregates tuples with a {@link GradoopId} as second field into + * tuples with a {@link GradoopIdSet} as second fields. + * + * @param type of first field + */ + +@FunctionAnnotation.ForwardedFields("f0") +public class AggregateGraphs + implements GroupReduceFunction, Tuple2> { + + /** + * Reduce object instantiations. + */ + private Tuple2 returnTuple = new Tuple2<>(); + + @Override + public void reduce(Iterable> iterable, + Collector> collector) throws Exception { + + if (returnTuple.f1 == null) { + returnTuple.f1 = new GradoopIdSet(); + } + + Iterator> it = iterable.iterator(); + Tuple2 first = it.next(); + returnTuple.f0 = first.f0; + returnTuple.f1.clear(); + returnTuple.f1.add(first.f1); + + while (it.hasNext()) { + returnTuple.f1.add(it.next().f1); + } + + collector.collect(returnTuple); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/CreateGraphHeadWithProperties.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/CreateGraphHeadWithProperties.java new file mode 100644 index 000000000000..23e149ec5aa2 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/CreateGraphHeadWithProperties.java @@ -0,0 +1,88 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.result.functions; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.types.Row; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.GraphHead; +import org.gradoop.common.model.impl.pojo.GraphHeadFactory; +import org.gradoop.common.model.impl.properties.Properties; + +import java.util.List; + +/** + * Create a graph head from a row while automatically setting properties. The fields between start + * and end are expected to be the properties in order of the property names, the last field of the + * row is expected to be a {@link GradoopId}. + */ +public class CreateGraphHeadWithProperties implements MapFunction { + + /** + * The start index of fields that will be set as properties of the graph heads. + */ + private int start; + + /** + * The end index of fields that will be set as properties of the graph heads. + */ + private int end; + + /** + * The factory used to create the graph heads. + */ + private GraphHeadFactory headFactory; + + /** + * List containing names for the properties to be set. + */ + private List propertyNames; + + /** + * Constructor. + * + * @param start start index of property fields + * @param end end index of property fields + * @param headFactory graph head factory + * @param propertyNames names of the properties to be set + */ + public CreateGraphHeadWithProperties( + int start, + int end, + GraphHeadFactory headFactory, + List propertyNames) { + + this.start = start; + this.end = end; + this.headFactory = headFactory; + this.propertyNames = propertyNames; + } + + @Override + public GraphHead map(Row row) { + + GraphHead head = headFactory.initGraphHead((GradoopId) row.getField(row.getArity() - 1)); + Properties props = new Properties(); + + for (int i = start; i < end; i++) { + props.set(propertyNames.get(i), row.getField(i)); + } + + head.setProperties(props); + + return head; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/PropertyDecoder.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/PropertyDecoder.java new file mode 100644 index 000000000000..77b24744339b --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/PropertyDecoder.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.result.functions; + +import org.apache.flink.api.common.functions.MapFunction; +import org.gradoop.common.model.api.entities.EPGMElement; +import org.gradoop.common.model.impl.properties.Property; +import org.gradoop.common.model.impl.properties.PropertyValue; + +import java.nio.charset.Charset; + +/** + * Decodes String representations of property types not yet supported by CAPF. + * + * @param an EPGMElement type + */ +public class PropertyDecoder implements MapFunction { + + @Override + public E map(E e) throws Exception { + if (e.getProperties() != null) { + for (Property prop : e.getProperties()) { + PropertyValue val = prop.getValue(); + if (val.isString()) { + if (val.getString().startsWith("CAPFProperty")) { + String encoded = val.getString().substring(12); + e.getProperties().set( + prop.getKey(), + PropertyValue.fromRawBytes(encoded.getBytes(Charset.forName("UTF-8")))); + } + } + } + } + return e; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/SplitRow.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/SplitRow.java new file mode 100644 index 000000000000..cbdc9fd0c644 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/SplitRow.java @@ -0,0 +1,66 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.result.functions; + +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; +import org.gradoop.common.model.impl.id.GradoopId; + +/** + * Function that splits a row into multiple tuples. For every field i between start and end + * a new tuple is created, such that the first field is equal to field i and the second field is + * equal to the last field of the row. The i-th field is expected to be a Long, the last field a + * {@link GradoopId}. + */ +public class SplitRow implements FlatMapFunction> { + + /** + * The start index of long fields that will be used to create new tuples. + */ + private int start; + + /** + * The end index of long fields that will be used to create new tuples. + */ + private int end; + + /** + * Reduce object instantiations. + */ + private Tuple2 reuseTuple = new Tuple2<>(); + + /** + * Constructor. + * + * @param start start index of long fields used to create new tuples + * @param end end index of long fields used to create new tuples + */ + public SplitRow(int start, int end) { + this.start = start; + this.end = end; + } + + @Override + public void flatMap(Row row, Collector> collector) throws Exception { + for (int i = start; i < end; i++) { + reuseTuple.f0 = (Long) row.getField(i); + reuseTuple.f1 = (GradoopId) row.getField(row.getArity() - 1); + collector.collect(reuseTuple); + } + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/package-info.java new file mode 100644 index 000000000000..e33e1706d9a7 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/functions/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Functions required for the handling of results returned by CAPF + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.result.functions; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/package-info.java new file mode 100644 index 000000000000..25d94c0d5c6d --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/cypher/capf/result/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains implementations for the handling of results returned by CAPF + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.result; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/Difference.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/Difference.java index 6a74b9b987c8..b6f007337262 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/Difference.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/Difference.java @@ -55,12 +55,4 @@ protected DataSet computeNewGraphHeads() { .groupBy(new IdOf0InTuple2()) .reduceGroup(new RemoveCut()); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return Difference.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/DifferenceBroadcast.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/DifferenceBroadcast.java index 5ced9da54541..c4c6af4ff7e4 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/DifferenceBroadcast.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/DifferenceBroadcast.java @@ -53,12 +53,4 @@ protected DataSet computeNewVertices( .withBroadcastSet(identifiers, GraphsContainmentFilterBroadcast.GRAPH_IDS); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return DifferenceBroadcast.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/functions/CreateTuple2WithLong.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/functions/CreateTuple2WithLong.java index fc4f17ebc036..4fa587743b5f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/functions/CreateTuple2WithLong.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/functions/CreateTuple2WithLong.java @@ -42,9 +42,6 @@ public CreateTuple2WithLong(Long secondField) { this.reuseTuple.f1 = secondField; } - /** - * {@inheritDoc} - */ @Override public Tuple2 map(O o) throws Exception { reuseTuple.f0 = o; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/functions/IdOf0InTuple2.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/functions/IdOf0InTuple2.java index 3f7c371f293b..cf517becd643 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/functions/IdOf0InTuple2.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/difference/functions/IdOf0InTuple2.java @@ -31,11 +31,8 @@ public class IdOf0InTuple2 implements KeySelector, GradoopId> { - /** - * {@inheritDoc} - */ @Override - public GradoopId getKey(Tuple2 pair) throws Exception { + public GradoopId getKey(Tuple2 pair) { return pair.f0.getId(); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/DistinctById.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/DistinctById.java index 655e8b06e091..a140338498ed 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/DistinctById.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/DistinctById.java @@ -32,9 +32,4 @@ public GraphCollection execute(GraphCollection collection) { collection.getVertices(), collection.getEdges()); } - - @Override - public String getName() { - return DistinctById.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/DistinctByIsomorphism.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/DistinctByIsomorphism.java index 126808c3356d..4c54c7dc8324 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/DistinctByIsomorphism.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/DistinctByIsomorphism.java @@ -49,9 +49,4 @@ public GraphCollection execute(GraphCollection collection) { return selectVerticesAndEdges(collection, graphHeads); } - - @Override - public String getName() { - return DistinctByIsomorphism.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/GroupByIsomorphism.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/GroupByIsomorphism.java index af8a62308558..1a605f52941d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/GroupByIsomorphism.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/distinction/GroupByIsomorphism.java @@ -75,9 +75,4 @@ protected DataSet getCanonicalLabels(GraphCollection collection return camBuilder .getGraphHeadStrings(collection); } - - @Override - public String getName() { - return GroupByIsomorphism.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/CollectionEquality.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/CollectionEquality.java index 95fc597582e3..49f268bc4269 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/CollectionEquality.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/CollectionEquality.java @@ -65,9 +65,4 @@ public DataSet execute(GraphCollection firstCollection, canonicalAdjacencyMatrixBuilder.execute(secondCollection) ); } - - @Override - public String getName() { - return this.getClass().getSimpleName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/CollectionEqualityByGraphIds.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/CollectionEqualityByGraphIds.java index 9c4f5fa595a4..762fa03378ca 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/CollectionEqualityByGraphIds.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/CollectionEqualityByGraphIds.java @@ -60,9 +60,4 @@ public DataSet execute(GraphCollection firstCollection, return Not.map(Or.reduce(d)); } - - @Override - public String getName() { - return this.getClass().getSimpleName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/GraphEquality.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/GraphEquality.java index 3d238a43d600..12ebde5a4923 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/GraphEquality.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/equality/GraphEquality.java @@ -64,9 +64,4 @@ public DataSet execute(LogicalGraph firstGraph, LogicalGraph secondGrap return collectionEquality .execute(collectionFactory.fromGraph(firstGraph), collectionFactory.fromGraph(secondGraph)); } - - @Override - public String getName() { - return this.getClass().getSimpleName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/exclusion/Exclusion.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/exclusion/Exclusion.java index 6fd0fd733822..7e345edfe2e8 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/exclusion/Exclusion.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/exclusion/Exclusion.java @@ -78,12 +78,4 @@ public LogicalGraph execute( return firstGraph.getConfig().getLogicalGraphFactory().fromDataSets(newVertexSet, newEdgeSet); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return Exclusion.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/exclusion/ReduceExclusion.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/exclusion/ReduceExclusion.java index caee4203783c..1c28de10c648 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/exclusion/ReduceExclusion.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/exclusion/ReduceExclusion.java @@ -75,9 +75,4 @@ public LogicalGraph execute(GraphCollection collection) { return collection.getConfig().getLogicalGraphFactory().fromDataSets(vertices, edges); } - - @Override - public String getName() { - return ReduceExclusion.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/fusion/VertexFusion.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/fusion/VertexFusion.java index fb6b40c77432..e19b8f877feb 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/fusion/VertexFusion.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/fusion/VertexFusion.java @@ -149,11 +149,4 @@ public LogicalGraph execute(LogicalGraph searchGraph, GraphCollection graphPatte return searchGraph.getConfig().getLogicalGraphFactory().fromDataSets(gh, vToRet, edges); } - - - - @Override - public String getName() { - return VertexFusion.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/Grouping.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/Grouping.java index e8e4d8672620..b2282ff2bce1 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/Grouping.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/Grouping.java @@ -130,9 +130,6 @@ public abstract class Grouping implements UnaryGraphToGraphOperator { this.edgeLabelGroups = edgeLabelGroups; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph execute(LogicalGraph graph) { LogicalGraph result; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/GroupingGroupCombine.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/GroupingGroupCombine.java index 8ee6125fed2e..11bbd9d86629 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/GroupingGroupCombine.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/GroupingGroupCombine.java @@ -121,9 +121,4 @@ protected LogicalGraph groupInternal(LogicalGraph graph) { return config.getLogicalGraphFactory().fromDataSets(superVertices, superEdges); } - - @Override - public String getName() { - return GroupingGroupCombine.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/GroupingGroupReduce.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/GroupingGroupReduce.java index fed773ce23c8..870bb852482e 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/GroupingGroupReduce.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/GroupingGroupReduce.java @@ -72,9 +72,6 @@ public class GroupingGroupReduce extends Grouping { useVertexLabels, useEdgeLabels, vertexLabelGroups, edgeLabelGroups); } - /** - * {@inheritDoc} - */ @Override protected LogicalGraph groupInternal(LogicalGraph graph) { @@ -104,12 +101,4 @@ protected LogicalGraph groupInternal(LogicalGraph graph) { return config.getLogicalGraphFactory().fromDataSets(superVertices, superEdges); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return GroupingGroupReduce.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildEdgeGroupItem.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildEdgeGroupItem.java index b99bef18d87d..354f3e225640 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildEdgeGroupItem.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildEdgeGroupItem.java @@ -50,9 +50,6 @@ public BuildEdgeGroupItem(boolean useLabel, List edgeLabelGroups) { this.reuseEdgeGroupItem = new EdgeGroupItem(); } - /** - * {@inheritDoc} - */ @Override public void flatMap(Edge edge, Collector collector) throws Exception { boolean usedEdgeLabelGroup = false; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildGroupItemBase.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildGroupItemBase.java index da83f46b4614..2c34273ea3b2 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildGroupItemBase.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildGroupItemBase.java @@ -78,6 +78,7 @@ public BuildGroupItemBase( * @param groupItem the group item to be set * @param element the epgm element * @param labelGroup label group to be assigned + * @throws IOException on failure */ protected void setGroupItem(GroupItem groupItem, EPGMElement element, LabelGroup labelGroup) throws IOException { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildSuperEdge.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildSuperEdge.java index 7a55d1e5b330..abaf5e986779 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildSuperEdge.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/BuildSuperEdge.java @@ -41,6 +41,7 @@ abstract class BuildSuperEdge extends BuildBase { * * @param edgeGroupItems edge group items * @return group representative item + * @throws IOException on failure */ protected EdgeGroupItem reduceInternal( Iterable edgeGroupItems) throws IOException { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/ReduceEdgeGroupItems.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/ReduceEdgeGroupItems.java index 4e3ca71e9949..43eef44c91e8 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/ReduceEdgeGroupItems.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/functions/ReduceEdgeGroupItems.java @@ -57,7 +57,7 @@ public ReduceEdgeGroupItems(boolean useLabel, EPGMEdgeFactory epgmEdgeFact * * @param edgeGroupItems edge group items * @param collector output collector - * @throws Exception + * @throws Exception on failure */ @Override public void reduce(Iterable edgeGroupItems, diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/tuples/LabelGroup.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/tuples/LabelGroup.java index cdb7b4e1888f..0da4cb961920 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/tuples/LabelGroup.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/grouping/tuples/LabelGroup.java @@ -252,6 +252,7 @@ public void aggregate(PropertyValueList values) { * * @param element attributed EPGM element * @return property values for aggregation + * @throws IOException on failure */ public PropertyValueList getIncrementValues(EPGMElement element) throws IOException { if (f3.isEmpty()) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/Intersection.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/Intersection.java index 334a969d8e0e..1320e00d2c3e 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/Intersection.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/Intersection.java @@ -43,12 +43,4 @@ protected DataSet computeNewGraphHeads() { .groupBy(new Id()) .reduceGroup(new GroupCountEquals(2)); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return Intersection.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/IntersectionBroadcast.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/IntersectionBroadcast.java index c310e6c5b4be..0400e4dca2f1 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/IntersectionBroadcast.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/IntersectionBroadcast.java @@ -44,9 +44,4 @@ protected DataSet computeNewVertices( .filter(new InAnyGraphBroadcast()) .withBroadcastSet(ids, GraphsContainmentFilterBroadcast.GRAPH_IDS); } - - @Override - public String getName() { - return IntersectionBroadcast.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/functions/GroupCountEquals.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/functions/GroupCountEquals.java index 3d76d70dd968..b84293a838a3 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/functions/GroupCountEquals.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/intersection/functions/GroupCountEquals.java @@ -50,7 +50,7 @@ public GroupCountEquals(long expectedGroupSize) { * * @param iterable graph data * @param collector output collector (contains 0 or 1 graph) - * @throws Exception + * @throws Exception on failure */ @Override public void reduce(Iterable iterable, diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/limit/Limit.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/limit/Limit.java index be4df74e04a6..fbe94b720cb2 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/limit/Limit.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/limit/Limit.java @@ -66,9 +66,4 @@ public GraphCollection execute(GraphCollection collection) { return collection.getConfig().getGraphCollectionFactory() .fromDataSets(graphHeads, filteredVertices, filteredEdges); } - - @Override - public String getName() { - return Limit.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/CypherPatternMatching.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/CypherPatternMatching.java index 2dc85bbd8e54..3af428f3bf79 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/CypherPatternMatching.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/CypherPatternMatching.java @@ -203,9 +203,4 @@ private EmbeddingMetaData computeNewMetaData(EmbeddingMetaData metaData, } return newMetaData; } - - @Override - public String getName() { - return CypherPatternMatching.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/filter/FilterEmbeddings.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/filter/FilterEmbeddings.java index ac78bc9c4dc8..970d8935bed0 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/filter/FilterEmbeddings.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/filter/FilterEmbeddings.java @@ -59,9 +59,7 @@ public FilterEmbeddings(DataSet input, CNF predicates, this.setName("FilterEmbeddings"); } - /** - * {@inheritDoc} - */ + @Override public DataSet evaluate() { return input .filter(new FilterEmbedding(predicates, metaData)) diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/preserving/explorative/ExplorativePatternMatching.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/preserving/explorative/ExplorativePatternMatching.java index a011602fa39d..fd819b029df4 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/preserving/explorative/ExplorativePatternMatching.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/preserving/explorative/ExplorativePatternMatching.java @@ -224,11 +224,6 @@ GradoopId.class, edgeStepJoinStrategy, vertexStepJoinStrategy, getVertexMapping( PostProcessor.extractGraphCollection(elements, graph.getConfig(), true); } - @Override - public String getName() { - return ExplorativePatternMatching.class.getName(); - } - /** * Used for configuring and creating a new {@link ExplorativePatternMatching} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/simulation/dual/DualSimulation.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/simulation/dual/DualSimulation.java index ee17b7769299..8c32c883a99a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/simulation/dual/DualSimulation.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/single/simulation/dual/DualSimulation.java @@ -294,12 +294,4 @@ private GraphCollection postProcess(LogicalGraph graph, return config.getGraphCollectionFactory().fromGraph( config.getLogicalGraphFactory().fromDataSets(matchVertices, matchEdges)); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return DualSimulation.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/TransactionalPatternMatching.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/TransactionalPatternMatching.java index 31c8098ffd63..956a09f06a0a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/TransactionalPatternMatching.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/TransactionalPatternMatching.java @@ -219,12 +219,4 @@ private GraphCollection findEmbeddings(GraphCollection collection, return collection.getConfig().getGraphCollectionFactory() .fromDataSets(newHeads, newVertices, newEdges); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return TransactionalPatternMatching.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/algorithm/DepthSearchMatching.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/algorithm/DepthSearchMatching.java index 71029b67c396..8985d28a412e 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/algorithm/DepthSearchMatching.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/algorithm/DepthSearchMatching.java @@ -83,9 +83,6 @@ public DepthSearchMatching() { this.edgeDict = new HashMap<>(); } - /** - * {@inheritDoc} - */ @Override public List> findEmbeddings(GraphWithCandidates graph, String query) { @@ -141,9 +138,6 @@ public List> findEmbeddings(GraphWithCandidates graph, return results; } - /** - * {@inheritDoc} - */ @Override public Boolean hasEmbedding(GraphWithCandidates graph, String query) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/function/InitGraphHeadWithLineage.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/function/InitGraphHeadWithLineage.java index 48adca35a134..2b12871093ff 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/function/InitGraphHeadWithLineage.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/matching/transactional/function/InitGraphHeadWithLineage.java @@ -45,9 +45,6 @@ public InitGraphHeadWithLineage(EPGMGraphHeadFactory epgmGraphHeadFac this.graphHeadFactory = epgmGraphHeadFactory; } - /** - * {@inheritDoc} - */ @Override public GraphHead map(Tuple2 idTuple) { GraphHead head = graphHeadFactory.initGraphHead(idTuple.f0); @@ -57,9 +54,6 @@ public GraphHead map(Tuple2 idTuple) { return head; } - /** - * {@inheritDoc} - */ @Override public TypeInformation getProducedType() { return TypeExtractor.createTypeInfo(graphHeadFactory.getType()); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/ReduceEdgeNeighborhood.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/ReduceEdgeNeighborhood.java index 59f3bb6c706e..0c09d4954473 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/ReduceEdgeNeighborhood.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/ReduceEdgeNeighborhood.java @@ -46,9 +46,6 @@ public ReduceEdgeNeighborhood(EdgeAggregateFunction function, EdgeDirection dire super(function, direction); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph execute(LogicalGraph graph) { DataSet vertices; @@ -89,12 +86,4 @@ public LogicalGraph execute(LogicalGraph graph) { return graph.getConfig().getLogicalGraphFactory().fromDataSets(graph.getGraphHead(), vertices, graph.getEdges()); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return ReduceEdgeNeighborhood.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/ReduceVertexNeighborhood.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/ReduceVertexNeighborhood.java index b385fe1b0332..e38ddbab1b74 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/ReduceVertexNeighborhood.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/ReduceVertexNeighborhood.java @@ -44,9 +44,6 @@ public ReduceVertexNeighborhood(VertexAggregateFunction function, EdgeDirection super(function, direction); } - /** - * {@inheritDoc} - */ @Override public LogicalGraph execute(LogicalGraph graph) { DataSet vertices; @@ -113,12 +110,4 @@ public LogicalGraph execute(LogicalGraph graph) { return graph.getConfig().getLogicalGraphFactory() .fromDataSets(graph.getGraphHead(), vertices, graph.getEdges()); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return ReduceVertexNeighborhood.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborEdgeFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborEdgeFunction.java index fefd31c80ca2..aff0f6841738 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborEdgeFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborEdgeFunction.java @@ -36,9 +36,6 @@ public NeighborEdgeFunction(EdgeAggregateFunction function) { this.function = function; } - /** - * {@inheritDoc} - */ @Override public EdgeAggregateFunction getFunction() { return function; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborEdgeReduceFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborEdgeReduceFunction.java index 66e46a3d5d3b..2d279a0adf51 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborEdgeReduceFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborEdgeReduceFunction.java @@ -40,9 +40,6 @@ public NeighborEdgeReduceFunction(EdgeAggregateFunction function) { super(function); } - /** - * {@inheritDoc} - */ @Override public void reduce(Iterable> tuples, Collector collector) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborVertexFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborVertexFunction.java index a20509d49ef9..dcec88f34771 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborVertexFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborVertexFunction.java @@ -36,9 +36,6 @@ public NeighborVertexFunction(VertexAggregateFunction function) { this.function = function; } - /** - * {@inheritDoc} - */ @Override public VertexAggregateFunction getFunction() { return function; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborVertexReduceFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborVertexReduceFunction.java index 22e3687dfe1d..fcee576b91fb 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborVertexReduceFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/NeighborVertexReduceFunction.java @@ -39,9 +39,6 @@ public NeighborVertexReduceFunction(VertexAggregateFunction function) { super(function); } - /** - * {@inheritDoc} - */ @Override public void reduce(Iterable> tuples, Collector collector) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/ShuffledVertexIdsFromEdge.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/ShuffledVertexIdsFromEdge.java index 1e702b83867f..3d1cea24fcf7 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/ShuffledVertexIdsFromEdge.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/ShuffledVertexIdsFromEdge.java @@ -40,9 +40,6 @@ public ShuffledVertexIdsFromEdge() { reuseTuple = new Tuple2(); } - /** - * {@inheritDoc} - */ @Override public void flatMap(Edge edge, Collector> collector) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexIdsFromEdge.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexIdsFromEdge.java index 824d98dfaf64..68b8fa58f52a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexIdsFromEdge.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexIdsFromEdge.java @@ -54,9 +54,6 @@ public VertexIdsFromEdge(boolean switched) { this.switched = switched; } - /** - * {@inheritDoc} - */ @Override public Tuple2 map(Edge edge) throws Exception { if (switched) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexIdsWithEdge.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexIdsWithEdge.java index 89f31be577b5..536825cefdea 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexIdsWithEdge.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexIdsWithEdge.java @@ -39,9 +39,6 @@ public VertexIdsWithEdge() { reuseTuple = new Tuple2(); } - /** - * {@inheritDoc} - */ @Override public void flatMap(Edge edge, Collector> collector) throws Exception { reuseTuple.setFields(edge.getSourceId(), edge); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexToFieldOne.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexToFieldOne.java index b089c7a9b2ec..d99a041134bd 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexToFieldOne.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/functions/VertexToFieldOne.java @@ -33,9 +33,6 @@ public class VertexToFieldOne */ private Tuple2 reuseTuple = new Tuple2(); - /** - * {@inheritDoc} - */ @Override public Tuple2 join(Tuple2 tuple, Vertex vertex) throws Exception { reuseTuple.setFields(tuple.f0, vertex); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/keyselector/IdInTuple.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/keyselector/IdInTuple.java index 7493a65d4c7a..edaa9bb24ccd 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/keyselector/IdInTuple.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/neighborhood/keyselector/IdInTuple.java @@ -41,9 +41,6 @@ public IdInTuple(int field) { this.field = field; } - /** - * {@inheritDoc} - */ @Override public GradoopId getKey(T tuple) throws Exception { return ((EPGMGraphElement) tuple.getField(field)).getId(); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/overlap/Overlap.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/overlap/Overlap.java index 356284a035c2..f40889ce6feb 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/overlap/Overlap.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/overlap/Overlap.java @@ -56,13 +56,5 @@ public LogicalGraph execute( return firstGraph.getConfig().getLogicalGraphFactory().fromDataSets(newVertices, newEdges); } - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return Overlap.class.getName(); - } - } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/overlap/ReduceOverlap.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/overlap/ReduceOverlap.java index d535b2be07bc..e5e1d7c393a1 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/overlap/ReduceOverlap.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/overlap/ReduceOverlap.java @@ -48,9 +48,4 @@ public LogicalGraph execute(GraphCollection collection) { getEdges(collection.getEdges(), graphIDs) ); } - - @Override - public String getName() { - return ReduceOverlap.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/rollup/EdgeRollUp.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/rollup/EdgeRollUp.java index d5c4041dc6bb..ea4696bb2f41 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/rollup/EdgeRollUp.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/rollup/EdgeRollUp.java @@ -67,9 +67,4 @@ LogicalGraph applyGrouping(LogicalGraph graph, List groupingKeys) { List> getGroupingKeyCombinations() { return createGroupingKeyCombinations(edgeGroupingKeys); } - - @Override - public String getName() { - return EdgeRollUp.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/rollup/VertexRollUp.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/rollup/VertexRollUp.java index de38a9e64e07..a2623b2b4f72 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/rollup/VertexRollUp.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/rollup/VertexRollUp.java @@ -68,9 +68,4 @@ LogicalGraph applyGrouping(LogicalGraph graph, List groupingKeys) { List> getGroupingKeyCombinations() { return createGroupingKeyCombinations(vertexGroupingKeys); } - - @Override - public String getName() { - return VertexRollUp.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/PageRankSampling.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/PageRankSampling.java index 960a60488205..fb8076313a00 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/PageRankSampling.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/PageRankSampling.java @@ -28,11 +28,12 @@ import org.gradoop.flink.model.impl.operators.aggregation.functions.max.MaxVertexProperty; import org.gradoop.flink.model.impl.operators.aggregation.functions.min.MinVertexProperty; import org.gradoop.flink.model.impl.operators.aggregation.functions.sum.SumVertexProperty; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; import org.gradoop.flink.model.impl.operators.sampling.functions.AddPageRankScoresToVertexCrossFunction; import org.gradoop.flink.model.impl.operators.sampling.functions.PageRankResultVertexFilter; /** - * Computes a PageRank-Sampling of the graph. + * Computes a PageRank-Sampling of the graph (new graph head will be generated). * * Uses the Gradoop-Wrapper of Flinks PageRank-algorithm {@link PageRank} with a dampening factor * and a number of maximum iterations. It computes a per-vertex score which is the sum of the @@ -98,7 +99,7 @@ public PageRankSampling(double dampeningFactor, int maxIteration, double thresho /** * {@inheritDoc} - * + *

* Vertices are sampled by using the Gradoop-Wrapper of Flinks PageRank-algorithm * {@link PageRank}. If they got different PageRank-scores, all scores are scaled * in a range between 0 and 1. @@ -113,15 +114,18 @@ public PageRankSampling(double dampeningFactor, int maxIteration, double thresho public LogicalGraph sample(LogicalGraph graph) { LogicalGraph pageRankGraph = new PageRank( - PAGE_RANK_SCORE_PROPERTY_KEY, dampeningFactor, maxIteration, true).execute(graph); + SamplingConstants.PAGE_RANK_SCORE_PROPERTY_KEY, + dampeningFactor, + maxIteration, + true).execute(graph); graph = graph.getConfig().getLogicalGraphFactory().fromDataSets( graph.getGraphHead(), pageRankGraph.getVertices(), pageRankGraph.getEdges()); graph = graph - .aggregate(new MinVertexProperty(PAGE_RANK_SCORE_PROPERTY_KEY), - new MaxVertexProperty(PAGE_RANK_SCORE_PROPERTY_KEY), - new SumVertexProperty(PAGE_RANK_SCORE_PROPERTY_KEY), + .aggregate(new MinVertexProperty(SamplingConstants.PAGE_RANK_SCORE_PROPERTY_KEY), + new MaxVertexProperty(SamplingConstants.PAGE_RANK_SCORE_PROPERTY_KEY), + new SumVertexProperty(SamplingConstants.PAGE_RANK_SCORE_PROPERTY_KEY), new VertexCount()); DataSet scaledVertices = graph.getVertices() @@ -138,17 +142,8 @@ public LogicalGraph sample(LogicalGraph graph) { .where(new TargetId<>()).equalTo(new Id<>()) .with(new LeftSide<>()); - graph = graph.getConfig().getLogicalGraphFactory().fromDataSets( - graph.getGraphHead(), scaledVertices, newEdges); + graph = graph.getFactory().fromDataSets(scaledVertices, newEdges); return graph; } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return PageRankSampling.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomEdgeSampling.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomEdgeSampling.java index b6788e793502..e2d6f5cb4868 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomEdgeSampling.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomEdgeSampling.java @@ -26,9 +26,9 @@ import org.gradoop.flink.model.impl.operators.sampling.functions.RandomFilter; /** - * Computes an edge sampling of the graph. Retains randomly chosen edges of a given relative amount - * and their associated source- and target-vertices. No unconnected vertices will retain in the - * sampled graph. + * Computes an edge sampling of the graph (new graph head will be generated). Retains randomly + * chosen edges of a given relative amount and their associated source- and target-vertices. No + * unconnected vertices will retain in the sampled graph. */ public class RandomEdgeSampling extends SamplingAlgorithm { /** @@ -62,9 +62,6 @@ public RandomEdgeSampling(float sampleSize, long randomSeed) { this.randomSeed = randomSeed; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph sample(LogicalGraph graph) { DataSet newEdges = graph.getEdges().filter(new RandomFilter<>(sampleSize, randomSeed)); @@ -82,14 +79,7 @@ public LogicalGraph sample(LogicalGraph graph) { .distinct(new Id<>()); DataSet newVertices = newSourceVertices.union(newTargetVertices).distinct(new Id<>()); - return graph.getConfig().getLogicalGraphFactory().fromDataSets(newVertices, newEdges); - } - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return RandomEdgeSampling.class.getName(); + return graph.getFactory().fromDataSets(newVertices, newEdges); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomLimitedDegreeVertexSampling.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomLimitedDegreeVertexSampling.java index ed3c58c6d34c..675d3c558c1a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomLimitedDegreeVertexSampling.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomLimitedDegreeVertexSampling.java @@ -25,13 +25,15 @@ import org.gradoop.flink.model.impl.functions.epgm.SourceId; import org.gradoop.flink.model.impl.functions.epgm.TargetId; import org.gradoop.flink.model.impl.functions.utils.LeftSide; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; import org.gradoop.flink.model.impl.operators.sampling.functions.LimitedDegreeVertexRandomFilter; import org.gradoop.flink.model.impl.operators.sampling.functions.VertexDegree; /** - * Computes a vertex sampling of the graph. Retains all vertices with a degree greater than a given - * degree threshold and degree type. Also retains randomly chosen vertices with a degree smaller - * or equal this threshold. Retains all edges which source- and target-vertices were chosen. + * Computes a vertex sampling of the graph (new graph head will be generated). Retains all vertices + * with a degree greater than a given degree threshold and degree type. Also retains randomly + * chosen vertices with a degree smaller or equal this threshold. Retains all edges which source- + * and target-vertices were chosen. */ public class RandomLimitedDegreeVertexSampling extends SamplingAlgorithm { @@ -111,24 +113,21 @@ public RandomLimitedDegreeVertexSampling(float sampleSize, long degreeThreshold, this.degreeType = degreeType; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph sample(LogicalGraph graph) { graph = new DistinctVertexDegrees( - DEGREE_PROPERTY_KEY, - IN_DEGREE_PROPERTY_KEY, - OUT_DEGREE_PROPERTY_KEY, + SamplingConstants.DEGREE_PROPERTY_KEY, + SamplingConstants.IN_DEGREE_PROPERTY_KEY, + SamplingConstants.OUT_DEGREE_PROPERTY_KEY, true).execute(graph); DataSet newVertices = graph.getVertices() .filter(new LimitedDegreeVertexRandomFilter<>( sampleSize, randomSeed, degreeThreshold, degreeType)) - .map(new PropertyRemover<>(DEGREE_PROPERTY_KEY)) - .map(new PropertyRemover<>(IN_DEGREE_PROPERTY_KEY)) - .map(new PropertyRemover<>(OUT_DEGREE_PROPERTY_KEY)); + .map(new PropertyRemover<>(SamplingConstants.DEGREE_PROPERTY_KEY)) + .map(new PropertyRemover<>(SamplingConstants.IN_DEGREE_PROPERTY_KEY)) + .map(new PropertyRemover<>(SamplingConstants.OUT_DEGREE_PROPERTY_KEY)); DataSet newEdges = graph.getEdges() .join(newVertices) @@ -138,16 +137,7 @@ public LogicalGraph sample(LogicalGraph graph) { .where(new TargetId<>()).equalTo(new Id<>()) .with(new LeftSide<>()); - return graph.getConfig().getLogicalGraphFactory() - .fromDataSets(graph.getGraphHead(), newVertices, newEdges); - } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return RandomLimitedDegreeVertexSampling.class.getName(); + return graph.getFactory().fromDataSets(newVertices, newEdges); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomNonUniformVertexSampling.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomNonUniformVertexSampling.java index 6c619879c571..90a02862f79d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomNonUniformVertexSampling.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomNonUniformVertexSampling.java @@ -25,15 +25,16 @@ import org.gradoop.flink.model.impl.functions.epgm.SourceId; import org.gradoop.flink.model.impl.functions.epgm.TargetId; import org.gradoop.flink.model.impl.functions.utils.LeftSide; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; import org.gradoop.flink.model.impl.operators.sampling.functions.AddMaxDegreeCrossFunction; import org.gradoop.flink.model.impl.operators.sampling.functions.NonUniformVertexRandomFilter; import org.gradoop.flink.model.impl.operators.sampling.functions.VertexToDegreeMap; /** - * Computes a vertex sampling of the graph. Retains randomly chosen vertices of a given relative - * amount and all edges which source- and target-vertices were chosen. A degree-dependent value - * is taken into account to have a bias towards high-degree vertices. There may retain some - * unconnected vertices in the sampled graph. + * Computes a vertex sampling of the graph (new graph head will be generated). Retains randomly + * chosen vertices of a given relative amount and all edges which source- and target-vertices + * were chosen. A degree-dependent value is taken into account to have a bias towards high-degree + * vertices. There may retain some unconnected vertices in the sampled graph. */ public class RandomNonUniformVertexSampling extends SamplingAlgorithm { @@ -68,35 +69,35 @@ public RandomNonUniformVertexSampling(float sampleSize, long randomSeed) { this.randomSeed = randomSeed; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph sample(LogicalGraph graph) { graph = new DistinctVertexDegrees( - DEGREE_PROPERTY_KEY, - IN_DEGREE_PROPERTY_KEY, - OUT_DEGREE_PROPERTY_KEY, + SamplingConstants.DEGREE_PROPERTY_KEY, + SamplingConstants.IN_DEGREE_PROPERTY_KEY, + SamplingConstants.OUT_DEGREE_PROPERTY_KEY, true).execute(graph); DataSet newVertices = graph.getVertices() - .map(new VertexToDegreeMap(DEGREE_PROPERTY_KEY)) + .map(new VertexToDegreeMap(SamplingConstants.DEGREE_PROPERTY_KEY)) .max(0) .cross(graph.getVertices()) - .with(new AddMaxDegreeCrossFunction(PROPERTY_KEY_MAX_DEGREE)); + .with(new AddMaxDegreeCrossFunction(SamplingConstants.PROPERTY_KEY_MAX_DEGREE)); graph = graph.getConfig().getLogicalGraphFactory() .fromDataSets(graph.getGraphHead(), newVertices, graph.getEdges()); newVertices = graph.getVertices().filter(new NonUniformVertexRandomFilter<>( - sampleSize, randomSeed, DEGREE_PROPERTY_KEY, PROPERTY_KEY_MAX_DEGREE)); + sampleSize, + randomSeed, + SamplingConstants.DEGREE_PROPERTY_KEY, + SamplingConstants.PROPERTY_KEY_MAX_DEGREE)); newVertices = newVertices - .map(new PropertyRemover<>(DEGREE_PROPERTY_KEY)) - .map(new PropertyRemover<>(IN_DEGREE_PROPERTY_KEY)) - .map(new PropertyRemover<>(OUT_DEGREE_PROPERTY_KEY)) - .map(new PropertyRemover<>(PROPERTY_KEY_MAX_DEGREE)); + .map(new PropertyRemover<>(SamplingConstants.DEGREE_PROPERTY_KEY)) + .map(new PropertyRemover<>(SamplingConstants.IN_DEGREE_PROPERTY_KEY)) + .map(new PropertyRemover<>(SamplingConstants.OUT_DEGREE_PROPERTY_KEY)) + .map(new PropertyRemover<>(SamplingConstants.PROPERTY_KEY_MAX_DEGREE)); DataSet newEdges = graph.getEdges() .join(newVertices) @@ -106,15 +107,6 @@ public LogicalGraph sample(LogicalGraph graph) { .where(new TargetId<>()).equalTo(new Id<>()) .with(new LeftSide<>()); - return graph.getConfig().getLogicalGraphFactory() - .fromDataSets(graph.getGraphHead(), newVertices, newEdges); - } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return RandomNonUniformVertexSampling.class.getName(); + return graph.getFactory().fromDataSets(newVertices, newEdges); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexEdgeSampling.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexEdgeSampling.java index 9d4045818644..be3242ff0f21 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexEdgeSampling.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexEdgeSampling.java @@ -18,10 +18,10 @@ import org.gradoop.flink.model.impl.epgm.LogicalGraph; /** - * Computes an edge sampling of the graph. First selects randomly chosen vertices of a given - * relative amount and all edges which source- and target-vertices were chosen. Then randomly - * chooses edges from this set of edges and their associated source- and target-vertices. - * No unconnected vertices will retain in the sampled graph. + * Computes an edge sampling of the graph (new graph head will be generated). First selects + * randomly chosen vertices of a given relative amount and all edges which source- and + * target-vertices were chosen. Then randomly chooses edges from this set of edges and their + * associated source- and target-vertices. No unconnected vertices will retain in the sampled graph. */ public class RandomVertexEdgeSampling extends SamplingAlgorithm { @@ -112,9 +112,6 @@ public RandomVertexEdgeSampling(float vertexSampleSize, float edgeSampleSize, lo this.vertexEdgeSamplingType = vertexEdgeSamplingType; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph sample(LogicalGraph graph) { @@ -137,12 +134,4 @@ public LogicalGraph sample(LogicalGraph graph) { return graph; } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return RandomVertexEdgeSampling.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexNeighborhoodSampling.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexNeighborhoodSampling.java index 4139fdfda08a..0ff0bf9f3050 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexNeighborhoodSampling.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexNeighborhoodSampling.java @@ -22,17 +22,18 @@ import org.gradoop.flink.model.impl.functions.epgm.Id; import org.gradoop.flink.model.impl.functions.epgm.SourceId; import org.gradoop.flink.model.impl.functions.tuple.Value0Of3; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; import org.gradoop.flink.model.impl.operators.sampling.functions.EdgeSourceVertexJoin; import org.gradoop.flink.model.impl.operators.sampling.functions.EdgeTargetVertexJoin; import org.gradoop.flink.model.impl.operators.sampling.functions.EdgesWithSampledVerticesFilter; import org.gradoop.flink.model.impl.operators.sampling.functions.FilterVerticesWithDegreeOtherThanGiven; import org.gradoop.flink.model.impl.operators.sampling.functions.Neighborhood; -import org.gradoop.flink.model.impl.operators.sampling.functions.VertexRandomMarkedMap; +import org.gradoop.flink.model.impl.operators.sampling.functions.RandomVertex; /** - * Computes a vertex sampling of the graph. Retains randomly chosen vertices of a given relative - * amount and includes all neighbors of those vertices in the sampling. All edges which source- - * and target-vertices were chosen are sampled, too. + * Computes a vertex sampling of the graph (new graph head will be generated). Retains randomly + * chosen vertices of a given relative amount and includes all neighbors of those vertices in the + * sampling. All edges which source- and target-vertices were chosen are sampled, too. */ public class RandomVertexNeighborhoodSampling extends SamplingAlgorithm { @@ -100,22 +101,19 @@ public RandomVertexNeighborhoodSampling(float sampleSize, this.neighborType = neighborType; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph sample(LogicalGraph graph) { DataSet sampledVertices = graph.getVertices() - .map(new VertexRandomMarkedMap(sampleSize, randomSeed, PROPERTY_KEY_SAMPLED)); + .map(new RandomVertex(sampleSize, randomSeed, SamplingConstants.PROPERTY_KEY_SAMPLED)); DataSet newEdges = graph.getEdges() .join(sampledVertices) .where(new SourceId<>()).equalTo(new Id<>()) - .with(new EdgeSourceVertexJoin(PROPERTY_KEY_SAMPLED)) + .with(new EdgeSourceVertexJoin(SamplingConstants.PROPERTY_KEY_SAMPLED)) .join(sampledVertices) .where(1).equalTo(new Id<>()) - .with(new EdgeTargetVertexJoin(PROPERTY_KEY_SAMPLED)) + .with(new EdgeTargetVertexJoin(SamplingConstants.PROPERTY_KEY_SAMPLED)) .filter(new EdgesWithSampledVerticesFilter(neighborType)) .map(new Value0Of3<>()); @@ -125,12 +123,4 @@ public LogicalGraph sample(LogicalGraph graph) { return graph; } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return RandomVertexNeighborhoodSampling.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexSampling.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexSampling.java index 1b7675185e48..8efde5d45de2 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexSampling.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexSampling.java @@ -26,9 +26,9 @@ import org.gradoop.flink.model.impl.operators.sampling.functions.RandomFilter; /** - * Computes a vertex sampling of the graph. Retains randomly chosen vertices of a given relative - * amount. Retains all edges which source- and target-vertices were chosen. There may retain some - * unconnected vertices in the sampled graph. + * Computes a vertex sampling of the graph (new graph head will be generated). Retains randomly + * chosen vertices of a given relative amount. Retains all edges which source- and + * target-vertices were chosen. There may retain some unconnected vertices in the sampled graph. */ public class RandomVertexSampling extends SamplingAlgorithm { /** @@ -62,9 +62,6 @@ public RandomVertexSampling(float sampleSize, long randomSeed) { this.randomSeed = randomSeed; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph sample(LogicalGraph graph) { @@ -79,14 +76,6 @@ public LogicalGraph sample(LogicalGraph graph) { .where(new TargetId<>()).equalTo(new Id<>()) .with(new LeftSide<>()); - return graph.getConfig().getLogicalGraphFactory().fromDataSets(newVertices, newEdges); - } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return RandomVertexSampling.class.getName(); + return graph.getFactory().fromDataSets(newVertices, newEdges); } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomWalkSampling.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomWalkSampling.java new file mode 100644 index 000000000000..2e1c95662892 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/RandomWalkSampling.java @@ -0,0 +1,108 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.sampling; + +import org.apache.flink.api.java.DataSet; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.flink.algorithms.gelly.randomjump.KRandomJumpGellyVCI; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.functions.epgm.ByProperty; +import org.gradoop.flink.model.impl.functions.epgm.Id; +import org.gradoop.flink.model.impl.functions.epgm.SourceId; +import org.gradoop.flink.model.impl.functions.tuple.Value0Of3; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; +import org.gradoop.flink.model.impl.operators.sampling.functions.EdgeSourceVertexJoin; +import org.gradoop.flink.model.impl.operators.sampling.functions.EdgeTargetVertexJoin; +import org.gradoop.flink.model.impl.operators.sampling.functions.EdgesWithSampledVerticesFilter; +import org.gradoop.flink.model.impl.operators.sampling.functions.Neighborhood; + +/** + * Computes a random walk sampling of the graph (new graph head will be generated). Retains visited + * vertices and edges where source and target vertex has been sampled. + */ +public class RandomWalkSampling extends SamplingAlgorithm { + + /** + * Sample size + */ + private final float sampleSize; + /** + * Number of start vertices + */ + private final int numberOfStartVertices; + /** + * Probability of jumping instead of walking along edges + */ + private final float jumpProbability; + /** + * Max iteration count + */ + private final int maxIteration; + + + /** + * Constructor to create an instance of RandomWalk sampling + * + * @param sampleSize sample size + * @param numberOfStartVertices number of start vertices + */ + public RandomWalkSampling(float sampleSize, int numberOfStartVertices) { + this.sampleSize = sampleSize; + this.numberOfStartVertices = numberOfStartVertices; + this.jumpProbability = 0.1f; + this.maxIteration = Integer.MAX_VALUE; + } + + /** + * Constructor to create an instance of RandomWalk sampling + * + * @param sampleSize sample size + * @param numberOfStartVertices number of start vertices + * @param jumpProbability probability to jump instead of walk + * @param maxIteration max gelly iteration count + */ + public RandomWalkSampling(float sampleSize, int numberOfStartVertices, + float jumpProbability, int maxIteration) { + + this.sampleSize = sampleSize; + this.numberOfStartVertices = numberOfStartVertices; + this.jumpProbability = jumpProbability; + this.maxIteration = maxIteration; + } + + @Override + protected LogicalGraph sample(LogicalGraph graph) { + + LogicalGraph gellyResult = new KRandomJumpGellyVCI(numberOfStartVertices, maxIteration, + jumpProbability, sampleSize).execute(graph); + + DataSet sampledVertices = gellyResult.getVertices() + .filter(new ByProperty<>(SamplingConstants.PROPERTY_KEY_SAMPLED)); + + DataSet sampledEdges = graph.getEdges() + .join(sampledVertices) + .where(new SourceId<>()).equalTo(new Id<>()) + .with(new EdgeSourceVertexJoin(SamplingConstants.PROPERTY_KEY_SAMPLED)) + .join(sampledVertices) + .where(1).equalTo(new Id<>()) + .with(new EdgeTargetVertexJoin(SamplingConstants.PROPERTY_KEY_SAMPLED)) + .filter(new EdgesWithSampledVerticesFilter(Neighborhood.BOTH)) + .map(new Value0Of3<>()); + + return graph.getFactory().fromDataSets(sampledVertices, sampledEdges); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/SamplingAlgorithm.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/SamplingAlgorithm.java index 47bd747c54b9..6ba7821b6cfc 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/SamplingAlgorithm.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/SamplingAlgorithm.java @@ -15,77 +15,14 @@ */ package org.gradoop.flink.model.impl.operators.sampling; -import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; -import org.gradoop.flink.model.impl.operators.sampling.functions.AddPageRankScoresToVertexCrossFunction; -import org.gradoop.flink.model.impl.operators.sampling.functions.FilterVerticesWithDegreeOtherThanGiven; -import org.gradoop.flink.model.impl.operators.sampling.functions.PageRankResultVertexFilter; -import org.gradoop.flink.model.impl.operators.sampling.functions.VertexDegree; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; /** - * Abstract class to provide property keys and methods for sampling algorithms. + * Abstract class to provide methods for sampling algorithms. */ public abstract class SamplingAlgorithm implements UnaryGraphToGraphOperator { - /** - * Key of degree property, used by {@link RandomNonUniformVertexSampling}, - * {@link RandomLimitedDegreeVertexSampling} and {@link FilterVerticesWithDegreeOtherThanGiven} - */ - public static final String DEGREE_PROPERTY_KEY = VertexDegree.BOTH.getName(); - - /** - * Key of in-degree property, used by {@link RandomNonUniformVertexSampling}, - * {@link RandomLimitedDegreeVertexSampling} and {@link FilterVerticesWithDegreeOtherThanGiven} - */ - public static final String IN_DEGREE_PROPERTY_KEY = VertexDegree.IN.getName(); - - /** - * Key of out-degree property, used by {@link RandomNonUniformVertexSampling}, - * {@link RandomLimitedDegreeVertexSampling} and {@link FilterVerticesWithDegreeOtherThanGiven} - */ - public static final String OUT_DEGREE_PROPERTY_KEY = VertexDegree.OUT.getName(); - - /** - * Key of the PageRankScore property used by {@link PageRankSampling} and - * {@link AddPageRankScoresToVertexCrossFunction} - */ - public static final String PAGE_RANK_SCORE_PROPERTY_KEY = "PageRankScore"; - - /** - * Key of the scaled_PageRankScore property used by {@link PageRankResultVertexFilter} and - * {@link AddPageRankScoresToVertexCrossFunction} - */ - public static final String SCALED_PAGE_RANK_SCORE_PROPERTY_KEY = "scaled_" + - PAGE_RANK_SCORE_PROPERTY_KEY; - - /** - * Key of the min_PageRankScore property used by {@link AddPageRankScoresToVertexCrossFunction} - */ - public static final String MIN_PAGE_RANK_SCORE_PROPERTY_KEY = "min_" + - PAGE_RANK_SCORE_PROPERTY_KEY; - - /** - * Key of the max_PageRankScore property used by {@link AddPageRankScoresToVertexCrossFunction} - */ - public static final String MAX_PAGE_RANK_SCORE_PROPERTY_KEY = "max_" + - PAGE_RANK_SCORE_PROPERTY_KEY; - - /** - * Key of the sum_PageRankScore property used by {@link AddPageRankScoresToVertexCrossFunction} - */ - public static final String SUM_PAGE_RANK_SCORE_PROPERTY_KEY = "sum_" + - PAGE_RANK_SCORE_PROPERTY_KEY; - - /** - * Key of a property generated by {@link RandomVertexNeighborhoodSampling} - */ - static final String PROPERTY_KEY_SAMPLED = "sampled"; - - /** - * Key of a property generated by {@link RandomNonUniformVertexSampling} - */ - static final String PROPERTY_KEY_MAX_DEGREE = "_maxDegree"; - @Override public LogicalGraph execute(LogicalGraph graph) { return sample(graph); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/SamplingConstants.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/SamplingConstants.java new file mode 100644 index 000000000000..e8e4d59d8e43 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/SamplingConstants.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.sampling.common; + +import org.gradoop.flink.model.impl.operators.sampling.PageRankSampling; +import org.gradoop.flink.model.impl.operators.sampling.RandomLimitedDegreeVertexSampling; +import org.gradoop.flink.model.impl.operators.sampling.RandomNonUniformVertexSampling; +import org.gradoop.flink.model.impl.operators.sampling.functions.AddPageRankScoresToVertexCrossFunction; +import org.gradoop.flink.model.impl.operators.sampling.functions.FilterVerticesWithDegreeOtherThanGiven; +import org.gradoop.flink.model.impl.operators.sampling.functions.PageRankResultVertexFilter; +import org.gradoop.flink.model.impl.operators.sampling.functions.VertexDegree; + +/** + * Constants for all sampling algorithms. + */ +public class SamplingConstants { + /** + * Key of degree property, used by {@link RandomNonUniformVertexSampling}, + * {@link RandomLimitedDegreeVertexSampling} and {@link FilterVerticesWithDegreeOtherThanGiven} + */ + public static final String DEGREE_PROPERTY_KEY = VertexDegree.BOTH.getName(); + + /** + * Key of in-degree property, used by {@link RandomNonUniformVertexSampling}, + * {@link RandomLimitedDegreeVertexSampling} and {@link FilterVerticesWithDegreeOtherThanGiven} + */ + public static final String IN_DEGREE_PROPERTY_KEY = VertexDegree.IN.getName(); + + /** + * Key of out-degree property, used by {@link RandomNonUniformVertexSampling}, + * {@link RandomLimitedDegreeVertexSampling} and {@link FilterVerticesWithDegreeOtherThanGiven} + */ + public static final String OUT_DEGREE_PROPERTY_KEY = VertexDegree.OUT.getName(); + + /** + * Key of a property generated by {@link RandomNonUniformVertexSampling} + */ + public static final String PROPERTY_KEY_MAX_DEGREE = "_maxDegree"; + + /** + * Key of the PageRankScore property used by {@link PageRankSampling} and + * {@link AddPageRankScoresToVertexCrossFunction} + */ + public static final String PAGE_RANK_SCORE_PROPERTY_KEY = "PageRankScore"; + + /** + * Key of the scaled_PageRankScore property used by {@link PageRankResultVertexFilter} and + * {@link AddPageRankScoresToVertexCrossFunction} + */ + public static final String SCALED_PAGE_RANK_SCORE_PROPERTY_KEY = "scaled_" + + PAGE_RANK_SCORE_PROPERTY_KEY; + + /** + * Key of the min_PageRankScore property used by {@link AddPageRankScoresToVertexCrossFunction} + */ + public static final String MIN_PAGE_RANK_SCORE_PROPERTY_KEY = "min_" + + PAGE_RANK_SCORE_PROPERTY_KEY; + + /** + * Key of the max_PageRankScore property used by {@link AddPageRankScoresToVertexCrossFunction} + */ + public static final String MAX_PAGE_RANK_SCORE_PROPERTY_KEY = "max_" + + PAGE_RANK_SCORE_PROPERTY_KEY; + + /** + * Key of the sum_PageRankScore property used by {@link AddPageRankScoresToVertexCrossFunction} + */ + public static final String SUM_PAGE_RANK_SCORE_PROPERTY_KEY = "sum_" + + PAGE_RANK_SCORE_PROPERTY_KEY; + + /** + * Property key used by multiple sampling algorithms + */ + public static final String PROPERTY_KEY_SAMPLED = "sampled"; + + /** + * Private constructor + */ + private SamplingConstants() { } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/SamplingEvaluationConstants.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/SamplingEvaluationConstants.java similarity index 84% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/SamplingEvaluationConstants.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/SamplingEvaluationConstants.java index be9095ca773e..2991f0e6a916 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/SamplingEvaluationConstants.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/SamplingEvaluationConstants.java @@ -13,7 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.flink.model.impl.operators.sampling.common; + +import org.gradoop.flink.model.impl.operators.statistics.AverageDegree; +import org.gradoop.flink.model.impl.operators.statistics.AverageIncomingDegree; +import org.gradoop.flink.model.impl.operators.statistics.AverageOutgoingDegree; +import org.gradoop.flink.model.impl.operators.statistics.ConnectedComponentsDistribution; +import org.gradoop.flink.model.impl.operators.statistics.GraphDensity; /** * Collected constants for sampling evaluation @@ -92,6 +98,11 @@ public class SamplingEvaluationConstants { */ public static final String FILE_TRIANGLE_COUNT = "triangle_count"; + /** + * Filename for saved degree centrality value of the graph + */ + public static final String FILE_DEGREE_CENTRALITY = "degree_centrality"; + /** * private Constructor */ diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/package-info.java new file mode 100644 index 000000000000..fb3a7a58f37e --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/common/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains a collection of graph sampling methods. + */ +package org.gradoop.flink.model.impl.operators.sampling.common; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/AddMaxDegreeCrossFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/AddMaxDegreeCrossFunction.java index 8725fb9df9cc..ca778072678c 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/AddMaxDegreeCrossFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/AddMaxDegreeCrossFunction.java @@ -38,9 +38,6 @@ public AddMaxDegreeCrossFunction(String nameOfMaxDegreeProperty) { this.nameOfMaxDegreeProperty = nameOfMaxDegreeProperty; } - /** - * {@inheritDoc} - */ @Override public Vertex cross(Tuple1 longTuple1, Vertex vertex) { vertex.setProperty(nameOfMaxDegreeProperty, longTuple1.f0); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/AddPageRankScoresToVertexCrossFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/AddPageRankScoresToVertexCrossFunction.java index 581b2aba17f7..23f14cdc1618 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/AddPageRankScoresToVertexCrossFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/AddPageRankScoresToVertexCrossFunction.java @@ -18,7 +18,7 @@ import org.apache.flink.api.common.functions.CrossFunction; import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.common.model.impl.pojo.Vertex; -import org.gradoop.flink.model.impl.operators.sampling.SamplingAlgorithm; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; /** * Writes the PageRank-scores stored in the graphHead to all vertices. @@ -32,22 +32,22 @@ public class AddPageRankScoresToVertexCrossFunction @Override public Vertex cross(Vertex vertex, GraphHead graphHead) { double min = graphHead.getPropertyValue( - SamplingAlgorithm.MIN_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); + SamplingConstants.MIN_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); double max = graphHead.getPropertyValue( - SamplingAlgorithm.MAX_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); + SamplingConstants.MAX_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); double sum = graphHead.getPropertyValue( - SamplingAlgorithm.SUM_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); + SamplingConstants.SUM_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); - vertex.setProperty(SamplingAlgorithm.MIN_PAGE_RANK_SCORE_PROPERTY_KEY, min); - vertex.setProperty(SamplingAlgorithm.MAX_PAGE_RANK_SCORE_PROPERTY_KEY, max); - vertex.setProperty(SamplingAlgorithm.SUM_PAGE_RANK_SCORE_PROPERTY_KEY, sum); + vertex.setProperty(SamplingConstants.MIN_PAGE_RANK_SCORE_PROPERTY_KEY, min); + vertex.setProperty(SamplingConstants.MAX_PAGE_RANK_SCORE_PROPERTY_KEY, max); + vertex.setProperty(SamplingConstants.SUM_PAGE_RANK_SCORE_PROPERTY_KEY, sum); vertex.setProperty("vertexCount", graphHead.getPropertyValue("vertexCount")); if (min != max) { double score = vertex.getPropertyValue( - SamplingAlgorithm.PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); + SamplingConstants.PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); vertex.setProperty( - SamplingAlgorithm.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY, (score - min) / (max - min)); + SamplingConstants.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY, (score - min) / (max - min)); } return vertex; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeSourceVertexJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeSourceVertexJoin.java index e721a52bf4d1..0dff1a5a83e3 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeSourceVertexJoin.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeSourceVertexJoin.java @@ -50,9 +50,6 @@ public EdgeSourceVertexJoin(String propertyKey) { this.propertyKey = propertyKey; } - /** - * {@inheritDoc} - */ @Override public Tuple3 join(Edge edge, Vertex vertex) throws Exception { reuse.f0 = edge; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeTargetVertexJoin.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeTargetVertexJoin.java index 5ea6ab5d2998..ed6387e73c73 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeTargetVertexJoin.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeTargetVertexJoin.java @@ -51,9 +51,6 @@ public EdgeTargetVertexJoin(String propertyKey) { this.propertyKey = propertyKey; } - /** - * {@inheritDoc} - */ @Override public Tuple3 join(Tuple3 interim, Vertex vertex) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeWithSourceTarget.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeWithSourceTarget.java index 9cdc1db5fbed..fafa7e91f1f3 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeWithSourceTarget.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgeWithSourceTarget.java @@ -36,9 +36,6 @@ public EdgeWithSourceTarget() { reuse = new Tuple3<>(); } - /** - * {@inheritDoc} - */ @Override public Tuple3 map(Edge edge) throws Exception { reuse.f0 = edge; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgesWithSampledVerticesFilter.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgesWithSampledVerticesFilter.java index fcb4eeba9308..a800a4bec79f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgesWithSampledVerticesFilter.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/EdgesWithSampledVerticesFilter.java @@ -41,9 +41,6 @@ public EdgesWithSampledVerticesFilter(Neighborhood neighborType) { this.neighborType = neighborType; } - /** - * {@inheritDoc} - */ @Override public boolean filter(Tuple3 tuple) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/FilterVerticesWithDegreeOtherThanGiven.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/FilterVerticesWithDegreeOtherThanGiven.java index c83b1a89c44c..3948ec664e8d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/FilterVerticesWithDegreeOtherThanGiven.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/FilterVerticesWithDegreeOtherThanGiven.java @@ -18,10 +18,10 @@ import org.apache.flink.api.java.DataSet; import org.gradoop.common.model.impl.pojo.Vertex; import org.gradoop.flink.algorithms.gelly.vertexdegrees.DistinctVertexDegrees; -import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.impl.functions.epgm.PropertyRemover; -import org.gradoop.flink.model.impl.operators.sampling.SamplingAlgorithm; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; /** * Retains all vertices which do not have the given degree. @@ -42,33 +42,22 @@ public FilterVerticesWithDegreeOtherThanGiven(long degree) { this.degree = degree; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph execute(LogicalGraph graph) { DistinctVertexDegrees distinctVertexDegrees = new DistinctVertexDegrees( - SamplingAlgorithm.DEGREE_PROPERTY_KEY, - SamplingAlgorithm.IN_DEGREE_PROPERTY_KEY, - SamplingAlgorithm.OUT_DEGREE_PROPERTY_KEY, + SamplingConstants.DEGREE_PROPERTY_KEY, + SamplingConstants.IN_DEGREE_PROPERTY_KEY, + SamplingConstants.OUT_DEGREE_PROPERTY_KEY, true); DataSet newVertices = distinctVertexDegrees.execute(graph).getVertices() - .filter(new VertexWithDegreeFilter<>(degree, SamplingAlgorithm.DEGREE_PROPERTY_KEY)) - .map(new PropertyRemover<>(SamplingAlgorithm.DEGREE_PROPERTY_KEY)) - .map(new PropertyRemover<>(SamplingAlgorithm.IN_DEGREE_PROPERTY_KEY)) - .map(new PropertyRemover<>(SamplingAlgorithm.OUT_DEGREE_PROPERTY_KEY)); + .filter(new VertexWithDegreeFilter<>(degree, SamplingConstants.DEGREE_PROPERTY_KEY)) + .map(new PropertyRemover<>(SamplingConstants.DEGREE_PROPERTY_KEY)) + .map(new PropertyRemover<>(SamplingConstants.IN_DEGREE_PROPERTY_KEY)) + .map(new PropertyRemover<>(SamplingConstants.OUT_DEGREE_PROPERTY_KEY)); return graph.getConfig().getLogicalGraphFactory().fromDataSets( graph.getGraphHead(), newVertices, graph.getEdges()); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return FilterVerticesWithDegreeOtherThanGiven.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/LimitedDegreeVertexRandomFilter.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/LimitedDegreeVertexRandomFilter.java index 3881ba62f451..b100723d2d7f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/LimitedDegreeVertexRandomFilter.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/LimitedDegreeVertexRandomFilter.java @@ -67,9 +67,6 @@ public LimitedDegreeVertexRandomFilter(float sampleSize, long randomSeed, long d this.degreeType = degreeType; } - /** - * {@inheritDoc} - */ @Override public boolean filter(V vertex) throws Exception { long degree = Long.parseLong(vertex.getPropertyValue(degreeType.getName()).toString()); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/NonUniformVertexRandomFilter.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/NonUniformVertexRandomFilter.java index bc8885fa7439..c82f003c4874 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/NonUniformVertexRandomFilter.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/NonUniformVertexRandomFilter.java @@ -51,24 +51,21 @@ public class NonUniformVertexRandomFilter * * @param sampleSize relative sample size * @param randomSeed random seed (can be 0) - * @param propertyNameOfDegree proeprty name of degree - * @param propertyNameofMaxDegree property name of max degree + * @param propertyNameOfDegree property name of degree + * @param propertyNameOfMaxDegree property name of max degree */ public NonUniformVertexRandomFilter(float sampleSize, long randomSeed, - String propertyNameOfDegree, String propertyNameofMaxDegree) { + String propertyNameOfDegree, String propertyNameOfMaxDegree) { threshold = sampleSize; randomGenerator = (randomSeed != 0L) ? new Random(randomSeed) : new Random(); this.propertyNameOfDegree = propertyNameOfDegree; - this.propertyNameofMaxDegree = propertyNameofMaxDegree; + this.propertyNameofMaxDegree = propertyNameOfMaxDegree; } - /** - * {@inheritDoc} - */ @Override public boolean filter(V vertex) throws Exception { - long degree = Long.parseLong(vertex.getPropertyValue(propertyNameOfDegree).toString()); - long maxDegree = Long.parseLong(vertex.getPropertyValue(propertyNameofMaxDegree).toString()); + long degree = vertex.getPropertyValue(propertyNameOfDegree).getLong(); + long maxDegree = vertex.getPropertyValue(propertyNameofMaxDegree).getLong(); return randomGenerator.nextFloat() <= (degree / (double) maxDegree) * threshold; } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/PageRankResultVertexFilter.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/PageRankResultVertexFilter.java index c83801b27e7b..2aa6fecd3425 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/PageRankResultVertexFilter.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/PageRankResultVertexFilter.java @@ -17,7 +17,7 @@ import org.apache.flink.api.common.functions.FilterFunction; import org.gradoop.common.model.impl.pojo.Vertex; -import org.gradoop.flink.model.impl.operators.sampling.PageRankSampling; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; /** * Retains all vertices with a PageRank-score greater or equal/smaller than a given @@ -59,8 +59,8 @@ public PageRankResultVertexFilter(double threshold, boolean sampleGreaterThanThr @Override public boolean filter(Vertex v) throws Exception { - if (v.hasProperty(PageRankSampling.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY)) { - double pr = v.getPropertyValue(PageRankSampling.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY) + if (v.hasProperty(SamplingConstants.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY)) { + double pr = v.getPropertyValue(SamplingConstants.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY) .getDouble(); if (sampleGreaterThanThreshold) { return pr > threshold; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/RandomFilter.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/RandomFilter.java index 48cf8f9ec4fe..a2b14050c87f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/RandomFilter.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/RandomFilter.java @@ -48,9 +48,6 @@ public RandomFilter(float sampleSize, long randomSeed) { randomGenerator = (randomSeed != 0L) ? new Random(randomSeed) : new Random(); } - /** - * {@inheritDoc} - */ @Override public boolean filter(E e) throws Exception { return randomGenerator.nextFloat() <= threshold; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexRandomMarkedMap.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/RandomVertex.java similarity index 81% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexRandomMarkedMap.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/RandomVertex.java index 652b266c6e1f..f782498f982f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexRandomMarkedMap.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/RandomVertex.java @@ -24,7 +24,7 @@ * Creates a random value for each vertex and marks those that are below a * given threshold. */ -public class VertexRandomMarkedMap implements MapFunction { +public class RandomVertex implements MapFunction { /** * Threshold to decide if a vertex needs to be filtered. */ @@ -36,30 +36,27 @@ public class VertexRandomMarkedMap implements MapFunction { /** * Property name for marking the vertex */ - private final String mark; + private final String samplingKey; /** * Creates a new filter instance. * * @param sampleSize relative sample size * @param randomSeed random seed (can be 0) - * @param mark the name of property for sampled vertices + * @param samplingKey the name of property for sampled vertices */ - public VertexRandomMarkedMap(float sampleSize, long randomSeed, String mark) { - this.mark = mark; + public RandomVertex(float sampleSize, long randomSeed, String samplingKey) { + this.samplingKey = samplingKey; this.sampleSize = sampleSize; this.randomGenerator = (randomSeed != 0L) ? new Random(randomSeed) : new Random(); } - /** - * {@inheritDoc} - */ @Override public Vertex map(Vertex vertex) throws Exception { if (randomGenerator.nextFloat() <= sampleSize) { - vertex.setProperty(mark, true); + vertex.setProperty(samplingKey, true); } else { - vertex.setProperty(mark, false); + vertex.setProperty(samplingKey, false); } return vertex; } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexWithDegreeFilter.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexWithDegreeFilter.java index 38f659f8badb..6f67f8dd0eb8 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexWithDegreeFilter.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/functions/VertexWithDegreeFilter.java @@ -44,9 +44,6 @@ public VertexWithDegreeFilter(long degree, String degreePropertyName) { this.degreePropertyName = degreePropertyName; } - /** - * {@inheritDoc} - */ @Override public boolean filter(V vertex) throws Exception { if (vertex.hasProperty(degreePropertyName)) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/selection/Selection.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/selection/Selection.java index 9b44bddb8c62..7673d887ff64 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/selection/Selection.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/selection/Selection.java @@ -78,9 +78,4 @@ private GraphCollection executeForTxLayout(GraphCollection collection) { return collection.getConfig().getGraphCollectionFactory() .fromTransactions(filteredTransactions); } - - @Override - public String getName() { - return Selection.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/selection/SelectionBase.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/selection/SelectionBase.java index 5443404b2ae6..0c26fd9b3ed8 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/selection/SelectionBase.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/selection/SelectionBase.java @@ -62,7 +62,4 @@ protected GraphCollection selectVerticesAndEdges( return collection.getConfig().getGraphCollectionFactory() .fromDataSets(graphHeads, vertices, edges); } - - @Override - public abstract String getName(); } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/Split.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/Split.java index 851b0cfdcf27..8ee6a3de7f5b 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/Split.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/Split.java @@ -67,9 +67,6 @@ public Split(Function> function) { this.function = function; } - /** - * {@inheritDoc} - */ @Override public GraphCollection execute(LogicalGraph graph) { @@ -146,12 +143,4 @@ public GraphCollection execute(LogicalGraph graph) { return graph.getConfig().getGraphCollectionFactory() .fromDataSets(newGraphs, vertices, edges); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return Split.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/AddNewGraphsToVertex.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/AddNewGraphsToVertex.java index e27a5215115b..1a71e54d6892 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/AddNewGraphsToVertex.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/AddNewGraphsToVertex.java @@ -31,9 +31,7 @@ @FunctionAnnotation.ReadFieldsSecond("f1") public class AddNewGraphsToVertex implements JoinFunction, V> { - /** - * {@inheritDoc} - */ + @Override public V join(V vertex, Tuple2 vertexWithGraphIds) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/JoinEdgeTupleWithSourceGraphs.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/JoinEdgeTupleWithSourceGraphs.java index 065438255fce..eeab9d5d2eb6 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/JoinEdgeTupleWithSourceGraphs.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/JoinEdgeTupleWithSourceGraphs.java @@ -38,9 +38,6 @@ public class JoinEdgeTupleWithSourceGraphs */ private final Tuple2 reuseTuple = new Tuple2<>(); - /** - * {@inheritDoc} - */ @Override public Tuple2 join( E left, Tuple2 right) { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/JoinEdgeTupleWithTargetGraphs.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/JoinEdgeTupleWithTargetGraphs.java index a10d20a82422..5eb1a1da5b4e 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/JoinEdgeTupleWithTargetGraphs.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/JoinEdgeTupleWithTargetGraphs.java @@ -41,9 +41,6 @@ public class JoinEdgeTupleWithTargetGraphs private final Tuple3 reuseTuple = new Tuple3<>(); - /** - * {@inheritDoc} - */ @Override public Tuple3 join( Tuple2 left, diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/SplitValues.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/SplitValues.java index 6290db4ca175..341576a6c4ff 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/SplitValues.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/split/functions/SplitValues.java @@ -52,9 +52,6 @@ public SplitValues(Function> function) { this.function = checkNotNull(function); } - /** - * {@inheritDoc} - */ @Override public void flatMap(V vertex, Collector> collector) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageClusteringCoefficient.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageClusteringCoefficient.java similarity index 90% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageClusteringCoefficient.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageClusteringCoefficient.java index dfe7ec1af3af..824196ee6942 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageClusteringCoefficient.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageClusteringCoefficient.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.flink.model.impl.operators.statistics; import org.apache.flink.api.java.DataSet; import org.gradoop.common.model.impl.pojo.GraphHead; @@ -23,7 +23,7 @@ import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; import org.gradoop.flink.model.impl.operators.aggregation.functions.count.VertexCount; import org.gradoop.flink.model.impl.operators.aggregation.functions.sum.SumVertexProperty; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.AddAverageCCValueToGraphHeadMap; +import org.gradoop.flink.model.impl.operators.statistics.functions.AddAverageCCValueToGraphHeadMap; /** * Calculates the average local clustering coefficient of a graph and writes it to the graph head. @@ -54,9 +54,4 @@ public LogicalGraph execute(LogicalGraph graph) { return graph.getConfig().getLogicalGraphFactory().fromDataSets( graphHead, graph.getVertices(), graph.getEdges()); } - - @Override - public String getName() { - return AverageClusteringCoefficient.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageDegree.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageDegree.java similarity index 81% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageDegree.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageDegree.java index 3a153771bcf0..7b7e14bddb8b 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageDegree.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageDegree.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.flink.model.impl.operators.statistics; import org.apache.flink.api.java.DataSet; import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; import org.gradoop.flink.model.impl.operators.aggregation.functions.count.VertexCount; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.AddSumDegreesToGraphHeadCrossFunction; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.CalculateAverageDegree; -import org.gradoop.flink.model.impl.operators.statistics.VertexDegrees; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.functions.AddSumDegreesToGraphHeadCrossFunction; +import org.gradoop.flink.model.impl.operators.statistics.functions.CalculateAverageDegree; /** * Calculates the average degree of a graph and writes it to the graph head. @@ -50,9 +50,4 @@ public LogicalGraph execute(LogicalGraph graph) { return graph.getConfig().getLogicalGraphFactory() .fromDataSets(newGraphHead, graph.getVertices(), graph.getEdges()); } - - @Override - public String getName() { - return AverageDegree.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageIncomingDegree.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageIncomingDegree.java similarity index 81% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageIncomingDegree.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageIncomingDegree.java index 6f3130a80087..1638620eee52 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageIncomingDegree.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageIncomingDegree.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.flink.model.impl.operators.statistics; import org.apache.flink.api.java.DataSet; import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; import org.gradoop.flink.model.impl.operators.aggregation.functions.count.VertexCount; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.AddSumDegreesToGraphHeadCrossFunction; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.CalculateAverageDegree; -import org.gradoop.flink.model.impl.operators.statistics.IncomingVertexDegrees; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.functions.AddSumDegreesToGraphHeadCrossFunction; +import org.gradoop.flink.model.impl.operators.statistics.functions.CalculateAverageDegree; /** * Calculates the average incoming degree of a graph and writes it to the graph head. @@ -50,9 +50,4 @@ public LogicalGraph execute(LogicalGraph graph) { return graph.getConfig().getLogicalGraphFactory() .fromDataSets(newGraphHead, graph.getVertices(), graph.getEdges()); } - - @Override - public String getName() { - return AverageIncomingDegree.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageOutgoingDegree.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageOutgoingDegree.java similarity index 81% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageOutgoingDegree.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageOutgoingDegree.java index 3ef7d359b1dd..1c9eb8caf92d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageOutgoingDegree.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/AverageOutgoingDegree.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.flink.model.impl.operators.statistics; import org.apache.flink.api.java.DataSet; import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; import org.gradoop.flink.model.impl.operators.aggregation.functions.count.VertexCount; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.AddSumDegreesToGraphHeadCrossFunction; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.CalculateAverageDegree; -import org.gradoop.flink.model.impl.operators.statistics.OutgoingVertexDegrees; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.functions.AddSumDegreesToGraphHeadCrossFunction; +import org.gradoop.flink.model.impl.operators.statistics.functions.CalculateAverageDegree; /** * Calculates the average outgoing degree of a graph and writes it to the graph head. @@ -50,9 +50,4 @@ public LogicalGraph execute(LogicalGraph graph) { return graph.getConfig().getLogicalGraphFactory() .fromDataSets(newGraphHead, graph.getVertices(), graph.getEdges()); } - - @Override - public String getName() { - return AverageOutgoingDegree.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ConnectedComponentsDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/ConnectedComponentsDistribution.java similarity index 88% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ConnectedComponentsDistribution.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/ConnectedComponentsDistribution.java index d7dc1bda15ec..353b0db786d5 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ConnectedComponentsDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/ConnectedComponentsDistribution.java @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.flink.model.impl.operators.statistics; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.tuple.Tuple3; import org.gradoop.flink.algorithms.gelly.connectedcomponents.AnnotateWeaklyConnectedComponents; import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.model.api.operators.UnaryGraphToValueOperator; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.AggregateListOfWccEdges; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.AggregateListOfWccVertices; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.GetConnectedComponentDistributionFlatMap; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.functions.AggregateListOfWccEdges; +import org.gradoop.flink.model.impl.operators.statistics.functions.AggregateListOfWccVertices; +import org.gradoop.flink.model.impl.operators.statistics.functions.GetConnectedComponentDistributionFlatMap; /** * Computes the weakly connected components of a graph. Uses the gradoop wrapper @@ -109,9 +110,4 @@ public DataSet> execute(LogicalGraph graph) { return graphWithWccIds.getGraphHead().flatMap( new GetConnectedComponentDistributionFlatMap(propertyKey, annotateEdges)); } - - @Override - public String getName() { - return ConnectedComponentsDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ValueConnectedComponentsDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/ConnectedComponentsDistributionAsValues.java similarity index 82% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ValueConnectedComponentsDistribution.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/ConnectedComponentsDistributionAsValues.java index e32a3fb416da..7f9bf1fae31f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ValueConnectedComponentsDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/ConnectedComponentsDistributionAsValues.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.flink.model.impl.operators.statistics; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.tuple.Tuple2; @@ -27,7 +27,7 @@ * * Returns a mapping of VertexId -> ComponentId */ -public class ValueConnectedComponentsDistribution +public class ConnectedComponentsDistributionAsValues implements UnaryGraphToValueOperator>> { /** @@ -40,23 +40,12 @@ public class ValueConnectedComponentsDistribution * * @param maxiIteration max iteration count. */ - public ValueConnectedComponentsDistribution(int maxiIteration) { + public ConnectedComponentsDistributionAsValues(int maxiIteration) { this.maxIteration = maxiIteration; } - /** - * {@inheritDoc} - */ @Override public DataSet> execute(LogicalGraph graph) { return new ValueWeaklyConnectedComponents(maxIteration).execute(graph); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return ValueConnectedComponentsDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DegreeCentrality.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DegreeCentrality.java new file mode 100644 index 000000000000..89da029259bb --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DegreeCentrality.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.statistics; + +import org.apache.flink.api.java.DataSet; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.flink.model.api.operators.UnaryGraphToValueOperator; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.functions.CalculateDegreeCentrality; +import org.gradoop.flink.model.impl.operators.statistics.functions.DegreeDistanceFunction; +import org.gradoop.flink.model.impl.tuples.WithCount; + +/** + * Computes the degree centrality of a graph + */ +public class DegreeCentrality extends DegreeCentralityBase + implements UnaryGraphToValueOperator> { + + /** + * Calculates the degree centrality of the graph + * using the function: (Sum(d(max) - d(i)) / (v_count -2) * (v_count-1) + * + * @param graph input graph + * @return DataSet with one degree centrality value + */ + @Override + public DataSet execute(LogicalGraph graph) { + + DataSet> degrees = new VertexDegrees().execute(graph); + DataSet vertexCount = new VertexCount().execute(graph); + + // for broadcasting + DataSet> maxDegree = degrees.max(1); + + return degrees + .map(new DegreeDistanceFunction(BROADCAST_NAME)) + .withBroadcastSet(maxDegree, BROADCAST_NAME) + .sum(0) + .crossWithTiny(vertexCount).with(new CalculateDegreeCentrality()); + } + + @Override + public String getName() { + return DegreeCentrality.class.getName(); + } +} + diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DegreeCentralityBase.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DegreeCentralityBase.java new file mode 100644 index 000000000000..7c89797532f9 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DegreeCentralityBase.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.statistics; + +/** + * Base class with constants for DegreeCentrality + */ +public abstract class DegreeCentralityBase { + /** + * Name for broadcasting variable + */ + static final String BROADCAST_NAME = "degree_max"; +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctEdgeProperties.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctEdgeProperties.java index 967eb3e35955..d961a98942f7 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctEdgeProperties.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctEdgeProperties.java @@ -33,9 +33,4 @@ public class DistinctEdgeProperties extends DistinctProperties { protected DataSet>> extractValuePairs(LogicalGraph graph) { return graph.getEdges().flatMap(new ExtractPropertyValues<>()); } - - @Override - public String getName() { - return DistinctEdgeProperties.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctEdgePropertiesByLabel.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctEdgePropertiesByLabel.java index eaef3f28c6b3..bcf549e02ef2 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctEdgePropertiesByLabel.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctEdgePropertiesByLabel.java @@ -35,9 +35,4 @@ protected DataSet, Set>> extractVal LogicalGraph graph) { return graph.getEdges().flatMap(new ExtractPropertyValuesByLabel<>()); } - - @Override - public String getName() { - return DistinctEdgePropertiesByLabel.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctSourceIds.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctSourceIds.java index 3ae027572197..d350fbe8f550 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctSourceIds.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctSourceIds.java @@ -34,9 +34,4 @@ public DataSet execute(LogicalGraph graph) { .distinct() ); } - - @Override - public String getName() { - return DistinctSourceIds.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctSourceIdsByEdgeLabel.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctSourceIdsByEdgeLabel.java index 76c2820529e2..8bfbf1a21564 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctSourceIdsByEdgeLabel.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctSourceIdsByEdgeLabel.java @@ -42,9 +42,4 @@ public DataSet> execute(LogicalGraph graph) { .sum(1) .map(new Tuple2ToWithCount<>()); } - - @Override - public String getName() { - return DistinctSourceIdsByEdgeLabel.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctTargetIds.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctTargetIds.java index 867495438c19..5edfa77ba31f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctTargetIds.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctTargetIds.java @@ -34,9 +34,4 @@ public DataSet execute(LogicalGraph graph) { .distinct() ); } - - @Override - public String getName() { - return DistinctTargetIds.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctTargetIdsByEdgeLabel.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctTargetIdsByEdgeLabel.java index b3b4d34aa2a8..e17d0c2da67a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctTargetIdsByEdgeLabel.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctTargetIdsByEdgeLabel.java @@ -42,9 +42,4 @@ public DataSet> execute(LogicalGraph graph) { .sum(1) .map(new Tuple2ToWithCount<>()); } - - @Override - public String getName() { - return DistinctTargetIdsByEdgeLabel.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctVertexProperties.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctVertexProperties.java index d44072943f31..ee32e4ffedfd 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctVertexProperties.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctVertexProperties.java @@ -33,9 +33,4 @@ public class DistinctVertexProperties extends DistinctProperties protected DataSet>> extractValuePairs(LogicalGraph graph) { return graph.getVertices().flatMap(new ExtractPropertyValues<>()); } - - @Override - public String getName() { - return DistinctVertexProperties.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctVertexPropertiesByLabel.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctVertexPropertiesByLabel.java index 62b53a2d4aaa..432b9d54dd10 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctVertexPropertiesByLabel.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/DistinctVertexPropertiesByLabel.java @@ -35,9 +35,4 @@ protected DataSet, Set>> extractVal LogicalGraph graph) { return graph.getVertices().flatMap(new ExtractPropertyValuesByLabel<>()); } - - @Override - public String getName() { - return DistinctVertexPropertiesByLabel.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeCount.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeCount.java index 52ae14df6337..b4750c04adbc 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeCount.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeCount.java @@ -29,9 +29,4 @@ public class EdgeCount implements UnaryGraphToValueOperator> { public DataSet execute(LogicalGraph graph) { return Count.count(graph.getEdges()); } - - @Override - public String getName() { - return EdgeCount.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeLabelDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeLabelDistribution.java index 6c204877a981..63babff33406 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeLabelDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeLabelDistribution.java @@ -34,9 +34,4 @@ public class EdgeLabelDistribution public DataSet> execute(LogicalGraph graph) { return new EdgeValueDistribution<>(new Label<>()).execute(graph); } - - @Override - public String getName() { - return EdgeLabelDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeValueDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeValueDistribution.java index 8652c724d99f..0d9b846387dd 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeValueDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/EdgeValueDistribution.java @@ -41,9 +41,4 @@ public EdgeValueDistribution(MapFunction valueFunction) { public DataSet> execute(LogicalGraph graph) { return compute(graph.getEdges()); } - - @Override - public String getName() { - return EdgeValueDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/GraphDensity.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/GraphDensity.java similarity index 85% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/GraphDensity.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/GraphDensity.java index 9e899c7614e6..ef8d59a2f590 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/GraphDensity.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/GraphDensity.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics; +package org.gradoop.flink.model.impl.operators.statistics; import org.apache.flink.api.java.DataSet; import org.gradoop.common.model.impl.pojo.GraphHead; @@ -21,7 +21,8 @@ import org.gradoop.flink.model.api.operators.UnaryGraphToGraphOperator; import org.gradoop.flink.model.impl.operators.aggregation.functions.count.EdgeCount; import org.gradoop.flink.model.impl.operators.aggregation.functions.count.VertexCount; -import org.gradoop.flink.model.impl.operators.sampling.statistics.functions.CalculateDensity; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.statistics.functions.CalculateDensity; /** * Computes the density of a graph and writes it to the graph head. @@ -29,9 +30,6 @@ */ public class GraphDensity implements UnaryGraphToGraphOperator { - /** - * {@inheritDoc} - */ @Override public LogicalGraph execute(LogicalGraph graph) { DataSet newGraphHead = graph @@ -42,9 +40,4 @@ public LogicalGraph execute(LogicalGraph graph) { return graph.getConfig().getLogicalGraphFactory() .fromDataSets(newGraphHead, graph.getVertices(), graph.getEdges()); } - - @Override - public String getName() { - return GraphDensity.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/IncomingVertexDegreeDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/IncomingVertexDegreeDistribution.java index f4129c8ddf30..6ff949dfb2d3 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/IncomingVertexDegreeDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/IncomingVertexDegreeDistribution.java @@ -37,9 +37,4 @@ public DataSet> execute(LogicalGraph graph) { .>project(1)) .map(new Tuple2ToWithCount<>()); } - - @Override - public String getName() { - return IncomingVertexDegreeDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/IncomingVertexDegrees.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/IncomingVertexDegrees.java index cdd80e008384..77419ccfee34 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/IncomingVertexDegrees.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/IncomingVertexDegrees.java @@ -37,9 +37,4 @@ public DataSet> execute(LogicalGraph graph) { .where(0).equalTo("*") .with(new SetOrCreateWithCount()); } - - @Override - public String getName() { - return IncomingVertexDegrees.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/OutgoingVertexDegreeDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/OutgoingVertexDegreeDistribution.java index e0b4ff8a5c44..d96bc7f584d7 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/OutgoingVertexDegreeDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/OutgoingVertexDegreeDistribution.java @@ -37,9 +37,4 @@ public DataSet> execute(LogicalGraph graph) { .>project(1)) .map(new Tuple2ToWithCount<>()); } - - @Override - public String getName() { - return OutgoingVertexDegreeDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/OutgoingVertexDegrees.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/OutgoingVertexDegrees.java index bced8bb37df3..de436f644f06 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/OutgoingVertexDegrees.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/OutgoingVertexDegrees.java @@ -37,9 +37,4 @@ public DataSet> execute(LogicalGraph graph) { .where(0).equalTo("*") .with(new SetOrCreateWithCount()); } - - @Override - public String getName() { - return OutgoingVertexDegrees.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/SourceLabelAndEdgeLabelDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/SourceLabelAndEdgeLabelDistribution.java index b26122dc037c..252c5186c5a0 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/SourceLabelAndEdgeLabelDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/SourceLabelAndEdgeLabelDistribution.java @@ -42,9 +42,4 @@ public DataSet>> execute(LogicalGraph graph) { .with(new BothLabels())) .map(new Tuple2ToWithCount<>()); } - - @Override - public String getName() { - return SourceLabelAndEdgeLabelDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/TargetLabelAndEdgeLabelDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/TargetLabelAndEdgeLabelDistribution.java index 7db8e2074520..839ec615bf1d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/TargetLabelAndEdgeLabelDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/TargetLabelAndEdgeLabelDistribution.java @@ -42,9 +42,4 @@ public DataSet>> execute(LogicalGraph graph) { .with(new BothLabels())) .map(new Tuple2ToWithCount<>()); } - - @Override - public String getName() { - return TargetLabelAndEdgeLabelDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexCount.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexCount.java index 05de84a9e32a..42c206766ad6 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexCount.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexCount.java @@ -29,9 +29,4 @@ public class VertexCount implements UnaryGraphToValueOperator> { public DataSet execute(LogicalGraph graph) { return Count.count(graph.getVertices()); } - - @Override - public String getName() { - return VertexCount.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexDegreeDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexDegreeDistribution.java index c54e9f86c902..d5906c0150eb 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexDegreeDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexDegreeDistribution.java @@ -37,9 +37,4 @@ public DataSet> execute(LogicalGraph graph) { .>project(1)) .map(new Tuple2ToWithCount<>()); } - - @Override - public String getName() { - return VertexDegreeDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexDegrees.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexDegrees.java index 702828f2d7a6..7606aef85280 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexDegrees.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexDegrees.java @@ -35,9 +35,4 @@ public DataSet> execute(LogicalGraph graph) { .where(0).equalTo(0) .with(new SumCounts<>()); } - - @Override - public String getName() { - return VertexDegrees.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexLabelDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexLabelDistribution.java index 07e3878e9f0e..ee1a06ea67b9 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexLabelDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexLabelDistribution.java @@ -34,9 +34,4 @@ public class VertexLabelDistribution public DataSet> execute(LogicalGraph graph) { return new VertexValueDistribution<>(new Label<>()).execute(graph); } - - @Override - public String getName() { - return VertexLabelDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexValueDistribution.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexValueDistribution.java index 5504a8775e1d..309c7847d1af 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexValueDistribution.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/VertexValueDistribution.java @@ -41,9 +41,4 @@ public VertexValueDistribution(MapFunction valueFunction) { public DataSet> execute(LogicalGraph graph) { return compute(graph.getVertices()); } - - @Override - public String getName() { - return VertexValueDistribution.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AddAverageCCValueToGraphHeadMap.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AddAverageCCValueToGraphHeadMap.java similarity index 97% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AddAverageCCValueToGraphHeadMap.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AddAverageCCValueToGraphHeadMap.java index 0c64f2582bd1..f433ffed8610 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AddAverageCCValueToGraphHeadMap.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AddAverageCCValueToGraphHeadMap.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics.functions; +package org.gradoop.flink.model.impl.operators.statistics.functions; import org.apache.flink.api.common.functions.MapFunction; import org.gradoop.common.model.impl.pojo.GraphHead; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AddSumDegreesToGraphHeadCrossFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AddSumDegreesToGraphHeadCrossFunction.java similarity index 96% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AddSumDegreesToGraphHeadCrossFunction.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AddSumDegreesToGraphHeadCrossFunction.java index d136c5a4d38c..428d6060f515 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AddSumDegreesToGraphHeadCrossFunction.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AddSumDegreesToGraphHeadCrossFunction.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics.functions; +package org.gradoop.flink.model.impl.operators.statistics.functions; import org.apache.flink.api.common.functions.CrossFunction; import org.gradoop.common.model.impl.id.GradoopId; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AggregateListOfWccEdges.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AggregateListOfWccEdges.java similarity index 92% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AggregateListOfWccEdges.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AggregateListOfWccEdges.java index 0753018a7ad9..989002b45851 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AggregateListOfWccEdges.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AggregateListOfWccEdges.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics.functions; +package org.gradoop.flink.model.impl.operators.statistics.functions; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.api.functions.EdgeAggregateFunction; @@ -48,7 +48,7 @@ public AggregateListOfWccEdges(String wccPropertyKey) { } @Override - public PropertyValue getIncrement(Element edge) { + public PropertyValue getIncrement(EPGMElement edge) { List valueList = new ArrayList<>(); valueList.add(PropertyValue.create(edge.getPropertyValue(wccPropertyKey).toString())); return PropertyValue.create(valueList); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AggregateListOfWccVertices.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AggregateListOfWccVertices.java similarity index 92% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AggregateListOfWccVertices.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AggregateListOfWccVertices.java index dd01498fcec7..f02f9c4b1a9e 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/AggregateListOfWccVertices.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/AggregateListOfWccVertices.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics.functions; +package org.gradoop.flink.model.impl.operators.statistics.functions; -import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.api.entities.EPGMElement; import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.api.functions.VertexAggregateFunction; @@ -48,7 +48,7 @@ public AggregateListOfWccVertices(String wccPropertyKey) { } @Override - public PropertyValue getIncrement(Element vertex) { + public PropertyValue getIncrement(EPGMElement vertex) { List valueList = new ArrayList<>(); valueList.add(PropertyValue.create(vertex.getPropertyValue(wccPropertyKey).toString())); return PropertyValue.create(valueList); diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/CalculateAverageDegree.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateAverageDegree.java similarity index 91% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/CalculateAverageDegree.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateAverageDegree.java index 99c0bc04819b..fc8ad83cee93 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/CalculateAverageDegree.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateAverageDegree.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics.functions; +package org.gradoop.flink.model.impl.operators.statistics.functions; import org.apache.flink.api.common.functions.MapFunction; import org.gradoop.common.model.impl.pojo.GraphHead; -import org.gradoop.flink.model.impl.operators.sampling.statistics.SamplingEvaluationConstants; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; /** * Calculates the average degree, depending on the sum-value of the degree type diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateDegreeCentrality.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateDegreeCentrality.java new file mode 100644 index 000000000000..5838dd6f5ae6 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateDegreeCentrality.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.statistics.functions; + +import org.apache.flink.api.common.functions.CrossFunction; +import org.apache.flink.api.java.tuple.Tuple1; + +/** + * Calculates the degree centrality of the graph + */ +public class CalculateDegreeCentrality implements CrossFunction, Long, Double> { + + /** + * Calculates the degree centrality of the graph + * using the function: (Sum(d(max) - d(i)) / (v_count -2) * (v_count-1) + * + * @param vertexDistanceSum sum of degree distances of the vertices + * @param vertexCount number of vertices + * @return degree centrality of graph + */ + @Override + public Double cross(Tuple1 vertexDistanceSum, Long vertexCount) { + double sum = vertexDistanceSum.f0; + return sum / ((vertexCount - 2) * (vertexCount - 1)); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/CalculateDensity.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateDensity.java similarity index 95% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/CalculateDensity.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateDensity.java index 11f32327cc25..ad8bd6d24585 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/CalculateDensity.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/CalculateDensity.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics.functions; +package org.gradoop.flink.model.impl.operators.statistics.functions; import org.apache.flink.api.common.functions.MapFunction; import org.gradoop.common.model.impl.pojo.GraphHead; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/DegreeDistanceFunction.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/DegreeDistanceFunction.java new file mode 100644 index 000000000000..f1680be0c7f3 --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/DegreeDistanceFunction.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.statistics.functions; + +import org.apache.flink.api.common.functions.RichMapFunction; +import org.apache.flink.api.java.tuple.Tuple1; +import org.apache.flink.configuration.Configuration; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.flink.model.impl.tuples.WithCount; + +/** + * Calculates the distance of max degree and the degree of each vertex + * + * {@inheritDoc} + */ +public class DegreeDistanceFunction extends RichMapFunction, Tuple1> { + + /** + * Storing max degree from broadcast + */ + private WithCount maxDegree; + + /** + * Name of broadcast variable + */ + private final String broadcastName; + + /** + * Tuple for reuse storing distances + */ + private Tuple1 reuse; + + /** + * Creates an instance of {@link DegreeDistanceFunction} + * to calculate the distance of max degree degree and the degree of each vertex + * + * @param broadcastName name of broadcast variable + */ + public DegreeDistanceFunction(String broadcastName) { + this.broadcastName = broadcastName; + this.reuse = new Tuple1<>(); + } + + /** + * Function called to store broadcasted max degree. + * + * @param parameter parameter + * @throws Exception throws any Exception + */ + @Override + public void open(Configuration parameter) throws Exception { + super.open(parameter); + maxDegree = getRuntimeContext() + .>getBroadcastVariable(this.broadcastName) + .get(0); + } + + /** + * Mapping function converting degree of vertex + * to distance of max degree and degree of vertex + * + * @param value degree of vertex + * @return degree distance + * @throws Exception throws any Exception + */ + @Override + public Tuple1 map(WithCount value) throws Exception { + reuse.f0 = maxDegree.f1 - value.f1; + return reuse; + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/GetConnectedComponentDistributionFlatMap.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/GetConnectedComponentDistributionFlatMap.java similarity index 97% rename from gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/GetConnectedComponentDistributionFlatMap.java rename to gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/GetConnectedComponentDistributionFlatMap.java index 60f098eface5..7a5340c0c9ef 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/sampling/statistics/functions/GetConnectedComponentDistributionFlatMap.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/functions/GetConnectedComponentDistributionFlatMap.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradoop.flink.model.impl.operators.sampling.statistics.functions; +package org.gradoop.flink.model.impl.operators.statistics.functions; import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.java.tuple.Tuple3; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctEdgePropertiesByLabelPreparer.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctEdgePropertiesByLabelPreparer.java index 8450f1bc04c5..54aab70092ad 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctEdgePropertiesByLabelPreparer.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctEdgePropertiesByLabelPreparer.java @@ -44,9 +44,4 @@ public class DistinctEdgePropertiesByLabelPreparer implements .map(value -> Tuple3.of(value.f0.f0, value.f0.f1, value.f1)) .returns(new TypeHint>() { }); } - - @Override - public String getName() { - return DistinctEdgePropertiesByLabelPreparer.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctSourceVertexCountPreparer.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctSourceVertexCountPreparer.java index f824fa574274..832e4be9005d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctSourceVertexCountPreparer.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctSourceVertexCountPreparer.java @@ -38,9 +38,4 @@ public MapOperator> execute(LogicalGraph graph) { .execute(graph) .map(new ObjectTo1<>()); } - - @Override - public String getName() { - return DistinctSourceVertexCountPreparer.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctTargetVertexCountPreparer.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctTargetVertexCountPreparer.java index f875700a78f4..41d39e944d6e 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctTargetVertexCountPreparer.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctTargetVertexCountPreparer.java @@ -38,9 +38,4 @@ public MapOperator> execute(final LogicalGraph graph) { .execute(graph) .map(new ObjectTo1<>()); } - - @Override - public String getName() { - return DistinctTargetVertexCountPreparer.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctVertexPropertiesByLabelPreparer.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctVertexPropertiesByLabelPreparer.java index aa5530e5fd38..dc379de6542d 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctVertexPropertiesByLabelPreparer.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/DistinctVertexPropertiesByLabelPreparer.java @@ -44,9 +44,4 @@ public class DistinctVertexPropertiesByLabelPreparer implements .map(value -> Tuple3.of(value.f0.f0, value.f0.f1, value.f1)) .returns(new TypeHint>() { }); } - - @Override - public String getName() { - return DistinctVertexPropertiesByLabelPreparer.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/EdgeCountPreparer.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/EdgeCountPreparer.java index 1cb407eebcfe..95cad52b703a 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/EdgeCountPreparer.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/EdgeCountPreparer.java @@ -38,9 +38,4 @@ public MapOperator> execute(final LogicalGraph graph) { .execute(graph) .map(new ObjectTo1<>()); } - - @Override - public String getName() { - return EdgeCountPreparer.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/SourceAndEdgeLabelDistributionPreparer.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/SourceAndEdgeLabelDistributionPreparer.java index bfc99033e85c..0423d794e976 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/SourceAndEdgeLabelDistributionPreparer.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/SourceAndEdgeLabelDistributionPreparer.java @@ -44,9 +44,4 @@ public class SourceAndEdgeLabelDistributionPreparer implements .map(value -> Tuple3.of(value.f0.f0, value.f0.f1, value.f1)) .returns(new TypeHint>() { }); } - - @Override - public String getName() { - return SourceAndEdgeLabelDistributionPreparer.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/TargetAndEdgeLabelDistributionPreparer.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/TargetAndEdgeLabelDistributionPreparer.java index 9ddd4bf4d47f..a26358ba1681 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/TargetAndEdgeLabelDistributionPreparer.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/TargetAndEdgeLabelDistributionPreparer.java @@ -44,9 +44,4 @@ public class TargetAndEdgeLabelDistributionPreparer implements .map(value -> Tuple3.of(value.f0.f0, value.f0.f1, value.f1)) .returns(new TypeHint>() { }); } - - @Override - public String getName() { - return TargetAndEdgeLabelDistributionPreparer.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/VertexCountPreparer.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/VertexCountPreparer.java index aa1767c5c1de..06bcebb5040b 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/VertexCountPreparer.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/statistics/writer/VertexCountPreparer.java @@ -38,9 +38,4 @@ public MapOperator> execute(final LogicalGraph graph) { .execute(graph) .map(new ObjectTo1<>()); } - - @Override - public String getName() { - return VertexCountPreparer.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/ApplySubgraph.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/ApplySubgraph.java index 85292db0ce48..e5ec9613896f 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/ApplySubgraph.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/ApplySubgraph.java @@ -85,9 +85,6 @@ public ApplySubgraph(FilterFunction vertexFilterFunction, this.edgeFilterFunction = edgeFilterFunction; } - /** - * {@inheritDoc} - */ @Override public GraphCollection executeForGVELayout(GraphCollection collection) { return vertexFilterFunction != null && edgeFilterFunction != null ? @@ -96,9 +93,6 @@ public GraphCollection executeForGVELayout(GraphCollection collection) { edgeInducedSubgraph(collection); } - /** - * {@inheritDoc} - */ @Override public GraphCollection executeForTxLayout(GraphCollection collection) { return executeForGVELayout(collection); @@ -353,9 +347,4 @@ private GraphCollection subgraph( return collection.getConfig().getGraphCollectionFactory() .fromDataSets(newGraphHeads, newVertices, newEdges); } - - @Override - public String getName() { - return ApplySubgraph.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/Subgraph.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/Subgraph.java index 61739d446b4a..813f6ae807f8 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/Subgraph.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/subgraph/Subgraph.java @@ -287,9 +287,4 @@ private LG verify(LG subgraph) { return subgraph.getFactory().fromDataSets(verifiedVertices, verifiedEdges); } - - @Override - public String getName() { - return Subgraph.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/transformation/Transformation.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/transformation/Transformation.java index b319bdb362bf..f09f1ae1e7aa 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/transformation/Transformation.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/transformation/Transformation.java @@ -117,9 +117,4 @@ protected LG executeInternal(DataSet graphHeads, DataSet vertices, DataSet return factory.fromDataSets(transformedGraphHeads, transformedVertices, transformedEdges); } - - @Override - public String getName() { - return Transformation.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/transformation/functions/TransformBase.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/transformation/functions/TransformBase.java index a25af36f1ba1..83a839fc4004 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/transformation/functions/TransformBase.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/transformation/functions/TransformBase.java @@ -48,7 +48,7 @@ protected TransformBase(TransformationFunction transformationFunction) { * * @param element current element * @return modified element - * @throws Exception + * @throws Exception on failure */ @Override public EL map(EL element) throws Exception { diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/union/Union.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/union/Union.java index ab43aa8ea67b..fd81bc8fd574 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/union/Union.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/union/Union.java @@ -28,9 +28,6 @@ */ public class Union extends SetOperatorBase { - /** - * {@inheritDoc} - */ @Override protected DataSet computeNewVertices( DataSet newGraphHeads) { @@ -39,9 +36,6 @@ protected DataSet computeNewVertices( .distinct(new Id()); } - /** - * {@inheritDoc} - */ @Override protected DataSet computeNewGraphHeads() { return firstCollection.getGraphHeads() @@ -49,21 +43,10 @@ protected DataSet computeNewGraphHeads() { .distinct(new Id()); } - /** - * {@inheritDoc} - */ @Override protected DataSet computeNewEdges(DataSet newVertices) { return firstCollection.getEdges() .union(secondCollection.getEdges()) .distinct(new Id()); } - - /** - * {@inheritDoc} - */ - @Override - public String getName() { - return Union.class.getName(); - } } diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/verify/Verify.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/verify/Verify.java new file mode 100644 index 000000000000..2793a3c4a06d --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/verify/Verify.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.verify; + +import org.apache.flink.api.java.DataSet; +import org.gradoop.common.model.api.entities.EPGMEdge; +import org.gradoop.common.model.api.entities.EPGMGraphHead; +import org.gradoop.common.model.api.entities.EPGMVertex; +import org.gradoop.flink.model.api.epgm.BaseGraph; +import org.gradoop.flink.model.api.operators.UnaryBaseGraphToBaseGraphOperator; +import org.gradoop.flink.model.impl.functions.epgm.Id; +import org.gradoop.flink.model.impl.functions.epgm.SourceId; +import org.gradoop.flink.model.impl.functions.epgm.TargetId; +import org.gradoop.flink.model.impl.functions.utils.LeftSide; + +/** + * Verifies a graph's edge set, removing dangling edges, i.e. edges with a source- or target-id + * not matching any vertices of this graph. + * + * @param The graph head type. + * @param The vertex type. + * @param The edge type. + * @param The graph type. + */ +public class Verify< + G extends EPGMGraphHead, + V extends EPGMVertex, + E extends EPGMEdge, + LG extends BaseGraph> implements UnaryBaseGraphToBaseGraphOperator { + + @Override + public LG execute(LG graph) { + DataSet vertices = graph.getVertices(); + DataSet verifiedEdges = graph.getEdges() + .join(vertices) + .where(new SourceId<>()) + .equalTo(new Id<>()) + .with(new LeftSide<>()) + .name("Verify Edges (1/2)") + .join(vertices) + .where(new TargetId<>()) + .equalTo(new Id<>()) + .with(new LeftSide<>()) + .name("Verify Edges (2/2)"); + return graph.getFactory().fromDataSets(graph.getGraphHead(), vertices, verifiedEdges); + } +} diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/verify/package-info.java b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/verify/package-info.java new file mode 100644 index 000000000000..3e430fe3969f --- /dev/null +++ b/gradoop-flink/src/main/java/org/gradoop/flink/model/impl/operators/verify/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Contains the verify operator used to remove dangling edges. + */ +package org.gradoop.flink.model.impl.operators.verify; diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/representation/transactional/RepresentationConverters.java b/gradoop-flink/src/main/java/org/gradoop/flink/representation/transactional/RepresentationConverters.java index bcca988432fc..e221417d733c 100644 --- a/gradoop-flink/src/main/java/org/gradoop/flink/representation/transactional/RepresentationConverters.java +++ b/gradoop-flink/src/main/java/org/gradoop/flink/representation/transactional/RepresentationConverters.java @@ -49,6 +49,7 @@ public class RepresentationConverters { * @param vertex data * * @return adjacency list + * @throws Exception on failure */ public static AdjacencyList getAdjacencyList( GraphTransaction transaction, diff --git a/gradoop-flink/src/main/resources/META-INF/NOTICE b/gradoop-flink/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..0a397ed92bd5 --- /dev/null +++ b/gradoop-flink/src/main/resources/META-INF/NOTICE @@ -0,0 +1,11 @@ +gradoop-flink +Copyright 2014-2019 Leipzig University (Database Research Group) + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- me.lemire.integercompression:JavaFastPFOR:0.1.10 +- me.lemire.integercompression:IntCompressor:0.1.10 +- me.lemire.integercompression:Simple16:0.1.10 +- org.apache.flink:flink-hbase_2.11:1.7.0 +- org.apache.flink:flink-java:1.7.0 +- org.apache.flink:flink-gelly_2.11:1.7.0 diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyClusteringCoefficientTestBase.java b/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyClusteringCoefficientTestBase.java index 8d66d2a81456..9a52e508e170 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyClusteringCoefficientTestBase.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/clusteringcoefficient/GellyClusteringCoefficientTestBase.java @@ -77,23 +77,31 @@ public void initGraphs() { /** * Checks if clustering coefficient properties are written + * + * @throws Exception on failure */ public abstract void validateGraphProperties(LogicalGraph graph) throws Exception; /** * Test for a fully connected graph + * + * @throws Exception on failure */ @Test public abstract void testFullyConnectedGraph() throws Exception; /** * Test for a graph with no connections + * + * @throws Exception on failure */ @Test public abstract void testNonConnectedGraph() throws Exception; /** * Test for a specific graph regarding directed vs. undirected + * + * @throws Exception on failure */ @Test public abstract void testSpecific() throws Exception; diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/labelpropagation/GradoopLabelPropagationTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/labelpropagation/GradoopLabelPropagationTest.java index 0e6a4c019c21..d7fba15c0739 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/labelpropagation/GradoopLabelPropagationTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/labelpropagation/GradoopLabelPropagationTest.java @@ -86,10 +86,9 @@ public void testByElementData() throws Exception { } /** - * Tests, if the resulting graph contains the same elements as the input - * graph. + * Tests, if the resulting graph contains the same elements as the input graph. * - * @throws Exception + * @throws Exception on failure */ @Test public void testByElementIds() throws Exception { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/randomjump/KRandomJumpGellyVCITest.java b/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/randomjump/KRandomJumpGellyVCITest.java new file mode 100644 index 000000000000..b16142d7a492 --- /dev/null +++ b/gradoop-flink/src/test/java/org/gradoop/flink/algorithms/gelly/randomjump/KRandomJumpGellyVCITest.java @@ -0,0 +1,180 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.algorithms.gelly.randomjump; + +import org.apache.flink.api.java.io.LocalCollectionOutputFormat; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test class for {@link KRandomJumpGellyVCI} + */ +public class KRandomJumpGellyVCITest extends GradoopFlinkTestBase { + /** + * The social graph used for testing + */ + private LogicalGraph socialGraph; + + /** + * The custom graph used for testing + */ + private LogicalGraph customGraph; + + /** + * List for result vertices + */ + private List resultVertices; + + /** + * Initialize graphs for testing + */ + @Before + public void initGraphs() throws Exception { + socialGraph = getSocialNetworkLoader().getLogicalGraph(); + String graphString = "graph[" + + "/* no edges graph */" + + "(v0 {id:0, value:\"A\"})" + + "(v1 {id:1, value:\"B\"})" + + "(v2 {id:2, value:\"C\"})" + + "]"; + customGraph = getLoaderFromString(graphString).getLogicalGraphByVariable("graph"); + } + + /** + * Test with social graph, with 1 starting vertex and at least half of the vertices to visit. + */ + @Test + public void baseTest() throws Exception { + LogicalGraph result = new KRandomJumpGellyVCI(1, 1000, 0.15, + 0.5).execute(socialGraph); + + commonValidation(socialGraph, result); + + long visitedVertices = resultVertices.stream().filter( + vertex -> vertex.getPropertyValue(SamplingConstants.PROPERTY_KEY_SAMPLED).getBoolean()) + .count(); + assertTrue("Wrong number of visited vertices, should be at least 6", + visitedVertices >= 6L); + } + + /** + * Test with social graph, with 3 starting vertices and at least half of the vertices to visit. + */ + @Test + public void base3StartVerticesTest() throws Exception { + LogicalGraph result = new KRandomJumpGellyVCI(3, 1000, 0.15, + 0.5).execute(socialGraph); + + commonValidation(socialGraph, result); + + long visitedVertices = resultVertices.stream().filter( + vertex -> vertex.getPropertyValue(SamplingConstants.PROPERTY_KEY_SAMPLED).getBoolean()) + .count(); + assertTrue("Wrong number of visited vertices, should be at least 6", + visitedVertices >= 6L); + } + + /** + * Test with social graph, with 1 starting vertex and all of the vertices to visit. + */ + @Test + public void visitAllTest() throws Exception { + LogicalGraph result = new KRandomJumpGellyVCI(1, 1000, 0.15, + 1.0).execute(socialGraph); + + commonValidation(socialGraph, result); + + resultVertices.forEach(vertex -> assertTrue( + "vertex " + vertex.getId() + " was not visited, all vertices should be", + vertex.getPropertyValue(SamplingConstants.PROPERTY_KEY_SAMPLED).getBoolean())); + } + + /** + * Test with social graph, with 3 starting vertices and all of the vertices to visit. + */ + @Test + public void visitAll3StartVerticesTest() throws Exception { + LogicalGraph result = new KRandomJumpGellyVCI(3, 1000, 0.15, + 1.0).execute(socialGraph); + + commonValidation(socialGraph, result); + + resultVertices.forEach(vertex -> assertTrue( + "vertex " + vertex.getId() + " was not visited, all vertices should be", + vertex.getPropertyValue(SamplingConstants.PROPERTY_KEY_SAMPLED).getBoolean())); + } + + /** + * Test with unconnected custom graph, with 1 starting vertex and all of the vertices to visit. + */ + @Test + public void visitAllJumpsOnlyTest() throws Exception { + LogicalGraph result = new KRandomJumpGellyVCI(1, 1000, 0.15, + 1.0).execute(customGraph); + + commonValidation(customGraph, result); + + resultVertices.forEach(vertex -> assertTrue( + "vertex " + vertex.getId() + " was not visited, all vertices should be", + vertex.getPropertyValue(SamplingConstants.PROPERTY_KEY_SAMPLED).getBoolean())); + } + + /** + * Validation for all test-cases. Writes the output, compares the vertex- and edge-count and + * checks the annotation with the visited-property. + * + * @param graph The original graph + * @param resultGraph The annotated result graph + */ + private void commonValidation(LogicalGraph graph, LogicalGraph resultGraph) throws Exception { + resultVertices = new ArrayList<>(); + List resultEdges = new ArrayList<>(); + resultGraph.getVertices().output(new LocalCollectionOutputFormat<>(resultVertices)); + resultGraph.getEdges().output(new LocalCollectionOutputFormat<>(resultEdges)); + getExecutionEnvironment().execute(); + + assertEquals("wrong number of vertices in resultGraph", + graph.getVertices().count(), resultGraph.getVertices().count()); + assertEquals("wrong number of edges in resultGraph", + graph.getEdges().count(), resultGraph.getEdges().count()); + resultVertices.forEach(vertex -> assertTrue("vertex " + vertex.getId() + " is not annotated", + vertex.hasProperty(SamplingConstants.PROPERTY_KEY_SAMPLED))); + resultEdges.forEach(edge -> assertTrue("edge " + edge.getId() + " is not annotated", + edge.hasProperty(SamplingConstants.PROPERTY_KEY_SAMPLED))); + + for (Edge edge : resultEdges) { + if (edge.getPropertyValue(SamplingConstants.PROPERTY_KEY_SAMPLED).getBoolean()) { + resultVertices.stream().filter(vertex -> vertex.getId().equals(edge.getSourceId())).forEach( + sourceVertex -> assertTrue("source of visited edge is not visited", + sourceVertex.getPropertyValue(SamplingConstants.PROPERTY_KEY_SAMPLED).getBoolean())); + resultVertices.stream().filter(vertex -> vertex.getId().equals(edge.getTargetId())).forEach( + targetVertex -> assertTrue("target of visited edge is not visited", + targetVertex.getPropertyValue(SamplingConstants.PROPERTY_KEY_SAMPLED).getBoolean())); + } + } + } +} diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVDataSinkTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVDataSinkTest.java index 73eba93c0a8b..50ccb3c48fc1 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVDataSinkTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVDataSinkTest.java @@ -254,7 +254,6 @@ public void testWriteMetadataCsvDistributed() throws Exception { BufferedReader br = new BufferedReader(new FileReader(metadataFile)); while ((line = br.readLine()) != null) { - System.out.println(line); checkMetadataCsvLine(line); } } diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/GradoopFlinkTestBase.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/GradoopFlinkTestBase.java index d738006a5259..22075249b39b 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/GradoopFlinkTestBase.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/GradoopFlinkTestBase.java @@ -137,7 +137,7 @@ protected void setCollectionLayoutFactory( * TODO: remove, when future issue is fixed * {@see http://mail-archives.apache.org/mod_mbox/flink-dev/201511.mbox/%3CCAC27z=PmPMeaiNkrkoxNFzoR26BOOMaVMghkh1KLJFW4oxmUmw@mail.gmail.com%3E} * - * @throws Exception + * @throws Exception on failure */ @BeforeClass public static void setupFlink() throws Exception { @@ -220,13 +220,11 @@ private FlinkAsciiGraphLoader getNewLoader() { // Test helper //---------------------------------------------------------------------------- - protected void collectAndAssertTrue(DataSet result) throws - Exception { + protected void collectAndAssertTrue(DataSet result) throws Exception { assertTrue("expected true", result.collect().get(0)); } - protected void collectAndAssertFalse(DataSet result) throws - Exception { + protected void collectAndAssertFalse(DataSet result) throws Exception { assertFalse("expected false", result.collect().get(0)); } diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/functions/epgm/LabelIsInTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/functions/epgm/LabelIsInTest.java new file mode 100644 index 000000000000..d8f9f8ce6afb --- /dev/null +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/functions/epgm/LabelIsInTest.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.functions.epgm; + +import org.gradoop.common.model.impl.pojo.Element; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.pojo.VertexFactory; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Test for the {@link LabelIsIn} filter function. + */ +public class LabelIsInTest extends GradoopFlinkTestBase { + + /** + * The comparator used to sort results. + */ + private Comparator comparator = Comparator.comparing(Element::getId); + + /** + * Some test vertices to filter. + */ + private List inputVertices; + + /** + * The expected vertices after the filter. + */ + private List expected; + + /** + * Initialize input vertices and expected result. + */ + @Before + public void setUp() { + VertexFactory vertexFactory = getConfig().getVertexFactory(); + Vertex v1 = vertexFactory.createVertex(); + Vertex v2 = vertexFactory.createVertex("a"); + Vertex v3 = vertexFactory.createVertex("b"); + Vertex v4 = vertexFactory.createVertex("c"); + inputVertices = Arrays.asList(v1, v2, v3, v4); + expected = Arrays.asList(v2, v3); + expected.sort(comparator); + } + + /** + * Test the filter using some elements. For this test the filter is created using the varargs + * constructor. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testFilterVarargs() throws Exception { + List result = getExecutionEnvironment().fromCollection(inputVertices) + .filter(new LabelIsIn<>("a", "b", "a", null)).collect(); + result.sort(comparator); + assertArrayEquals(expected.toArray(), result.toArray()); + } + + /** + * Test the filter using some elements. For this test the filter is created using the other + * constructor. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testFilterCollection() throws Exception { + List result = getExecutionEnvironment().fromCollection(inputVertices) + .filter(new LabelIsIn<>(Arrays.asList("a", "b", "a", null))).collect(); + result.sort(comparator); + assertArrayEquals(expected.toArray(), result.toArray()); + } +} diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/aggregation/ApplyAggregationTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/aggregation/ApplyAggregationTest.java index 805640fc7a8a..ab2d3e5de497 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/aggregation/ApplyAggregationTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/aggregation/ApplyAggregationTest.java @@ -41,9 +41,6 @@ import org.junit.Assert; import org.junit.Test; -import java.math.BigDecimal; -import java.math.MathContext; -import java.math.RoundingMode; import java.util.List; import static org.junit.Assert.assertEquals; @@ -321,14 +318,8 @@ public void testWithMixedTypePropertyValues() throws Exception { if (graphHead.getId().equals(g0Id)) { assertEquals(1.5f, vertexAggregate.getFloat(), 0.00001); - assertEquals( - new BigDecimal("4.0"), - edgeAggregate.getBigDecimal() - .round(new MathContext(2, RoundingMode.HALF_UP))); - assertEquals( - new BigDecimal("5.5"), - elementAggregate.getBigDecimal() - .round(new MathContext(2, RoundingMode.HALF_UP))); + assertEquals(4.0f, edgeAggregate.getFloat(), 0.00001); + assertEquals(5.5f, elementAggregate.getFloat(), 0.00001); } else if (graphHead.getId().equals(g1Id)) { assertEquals(0.5f, vertexAggregate.getFloat(), 0.00001); assertEquals(2L, edgeAggregate.getLong()); diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/TestData.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/TestData.java new file mode 100644 index 000000000000..7d0176c85e7a --- /dev/null +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/TestData.java @@ -0,0 +1,177 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf; + +public class TestData { + + public static final String DATA_GRAPH_VARIABLE = "db"; + + public static final String CHAIN_PATTERN_0 = + "MATCH (n1:A)-[e1:a]->(n2:B) RETURN *"; + + public static final String CHAIN_PATTERN_1 = + "MATCH (n1:A)-[e1:a]->(n2:A) RETURN *"; + + public static final String CHAIN_PATTERN_2 = + "MATCH (n1:A) RETURN *"; + + public static final String CHAIN_PATTERN_3 = + "MATCH (n1:B)-[e1:d]->(n2:B) RETURN *"; + + public static final String CHAIN_PATTERN_4 = + "MATCH (n1:A)-[e1:a]->(n2:A)-[e2:a]->(n3:A) RETURN *"; + + public static final String CHAIN_PATTERN_5 = + "MATCH (n1:B)-[e1:b]->(n2:C)<-[e2:a]-(n3:A) RETURN *"; + + public static final String CHAIN_PATTERN_6 = + "MATCH (c)<--(a)-->(b) RETURN *"; + + public static final String CHAIN_PATTERN_7 = + "MATCH (a)-->(b),(a)-->(c) RETURN *"; + + public static final String LOOP_PATTERN_0 = + "MATCH (b:B)-[e1:d]->(b) RETURN *"; + + public static final String CYCLE_PATTERN_0 = + "MATCH (a:A)-[e1:a]->(b:B)-[e2:a]->(a) RETURN *"; + + public static final String CYCLE_PATTERN_1 = + "MATCH (b:B)-[e1:d]->(n1:B)-[e2:d]->(b) RETURN *"; + + public static final String CYCLE_PATTERN_2 = + "MATCH (a:A)-[e1:a]->(b:B)-[e2:a]->(a),(b)-[e3:b]->(n1:C) RETURN *"; + + public static String CYCLE_PATTERN_3 = + "MATCH (a:A)-[e1:a]->(b:B)-[e2:a]->(a),(b)-[e3:b]->(n1:C)<-[e4:a]-(n1:B) RETURN *"; + + public static final String CYCLE_PATTERN_4 = + "MATCH (a0:A)-[e1:a]->(b0:B)-[e2:a]->(a1:A)-[e3:a]->(b1:B)-[e4:a]->(a0) RETURN *"; + + public static final String CYCLE_PATTERN_5 = + "MATCH (v0:B)-[e1:a]->(v1:C)<-[e2:b]-(v0) RETURN *"; + + public static final String CYCLE_PATTERN_6 = + "MATCH (v0:A)-[e1:a]->(v1:A)<-[e2:a]-(v0) RETURN *"; + + public static final String UNLABELED_PATTERN_0 = + "MATCH (n1) RETURN *"; + + public static final String UNLABELED_PATTERN_1 = + "MATCH (n1)-->(n2) RETURN *"; + + public static final String UNLABELED_PATTERN_2 = + "MATCH (n1)-[e1:b]->(n2) RETURN *"; + + public static final String UNLABELED_PATTERN_3 = + "MATCH (n1:A)-->(n2:B) RETURN *"; + + public static final String VAR_LENGTH_PATH_PATTERN_0 = + "MATCH (n1:B)-[e1:a*2..2]->(n2:B) RETURN *"; + + public static final String VAR_LENGTH_PATH_PATTERN_1 = + "MATCH (n1:B)<-[e1:a*2..2]-(n2:B) RETURN *"; + + public static final String VAR_LENGTH_PATH_PATTERN_2 = + "MATCH (n1:B)-[e1:d*2..3]->(n2) RETURN *"; + + public static final String VAR_LENGTH_PATH_PATTERN_3 = + "MATCH (n1:A)-[e1:a*]->(n2) RETURN *"; + + public static final String VAR_LENGTH_PATH_PATTERN_4 = + "MATCH (s:A)-[e1:a*1..2]->(s) RETURN *"; + + public static final String GRAPH_1 = DATA_GRAPH_VARIABLE + + "[" + + "(v0:B {id : 0})" + + "(v1:A {id : 1})" + + "(v2:A {id : 2})" + + "(v3:C {id : 3})" + + "(v4:B {id : 4})" + + "(v5:A {id : 5})" + + "(v6:B {id : 6})" + + "(v7:C {id : 7})" + + "(v8:B {id : 8})" + + "(v9:C {id : 9})" + + "(v10:D {id : 10})" + + "(v0)-[e0:a {id : 0}]->(v1)" + + "(v0)-[e1:a {id : 1}]->(v3)" + + "(v1)-[e2:a {id : 2}]->(v6)" + + "(v2)-[e3:a {id : 3}]->(v6)" + + "(v4)-[e4:a {id : 4}]->(v1)" + + "(v4)-[e5:b {id : 5}]->(v3)" + + "(v5)-[e6:a {id : 6}]->(v4)" + + "(v6)-[e7:a {id : 7}]->(v2)" + + "(v6)-[e8:a {id : 8}]->(v5)" + + "(v6)-[e9:b {id : 9}]->(v7)" + + "(v8)-[e10:a {id : 10}]->(v5)" + + "(v5)-[e11:a {id : 11}]->(v9)" + + "(v9)-[e12:c {id : 12}]->(v10)" + + "]"; + + public static final String GRAPH_2 = DATA_GRAPH_VARIABLE + + "[" + + "(v0:B {id : 0})" + + "(v1:A {id : 1})" + + "(v2:A {id : 2})" + + "(v3:A {id : 3})" + + "(v4:C {id : 4})" + + "(v5:B {id : 5})" + + "(v6:B {id : 6})" + + "(v7:C {id : 7})" + + "(v8:B {id : 8})" + + "(v9:B {id : 9})" + + "(v10:A {id : 10})" + + "(v11:C {id : 11})" + + "(v12:D {id : 12})" + + "(v1)-[e0:a {id : 0}]->(v0)" + + "(v0)-[e1:b {id : 1}]->(v4)" + + "(v0)-[e2:a {id : 2}]->(v4)" + + "(v0)-[e3:a {id : 3}]->(v3)" + + "(v3)-[e4:a {id : 4}]->(v5)" + + "(v5)-[e5:a {id : 5}]->(v1)" + + "(v1)-[e6:a {id : 6}]->(v6)" + + "(v6)-[e7:a {id : 7}]->(v2)" + + "(v2)-[e8:a {id : 8}]->(v6)" + + "(v5)-[e9:a {id : 9}]->(v4)" + + "(v5)-[e10:b {id : 10}]->(v4)" + + "(v6)-[e11:b {id : 11}]->(v7)" + + "(v8)-[e12:a {id : 12}]->(v7)" + + "(v10)-[e13:a {id : 13}]->(v5)" + + "(v6)-[e14:a {id : 14}]->(v10)" + + "(v9)-[e15:d {id : 15}]->(v9)" + + "(v9)-[e16:a {id : 16}]->(v10)" + + "(v10)-[e17:d {id : 17}]->(v11)" + + "(v11)-[e18:a {id : 18}]->(v12)" + + "]"; + + public static final String GRAPH_3 = DATA_GRAPH_VARIABLE + + "[" + + "(v0:A {id : 0})-[e0:a {id : 0}]->(v1:A {id : 1})" + + "(v1)-[e1:a {id : 1}]->(v2:A {id : 2})" + + "(v2)-[e2:a {id : 2}]->(v3:A {id : 3})" + + "]"; + + public static final String GRAPH_4 = DATA_GRAPH_VARIABLE + + "[" + + "(v0:A {id : 0})-[e0:a {id : 0}]->(v1:A {id : 1})" + + "(v1)-[e1:a {id : 1}]->(v2:A {id : 2})-[e3:a {id : 3}]->(v3:A {id : 3})" + + "(v1)-[e2:a {id : 2}]->(v2)" + + "]"; + + public static final String GRAPH_5 = DATA_GRAPH_VARIABLE + + "[(v0 {id : 0})-[e0 {id:0}]->(v1 {id : 1})]"; +} diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFPatternMatchingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFPatternMatchingTest.java new file mode 100644 index 000000000000..c3e5c165c055 --- /dev/null +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFPatternMatchingTest.java @@ -0,0 +1,292 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.sinks.BatchTableSink; +import org.apache.flink.types.Row; +import org.gradoop.common.model.impl.metadata.MetaData; +import org.gradoop.common.model.impl.metadata.PropertyMetaData; +import org.gradoop.flink.io.impl.csv.metadata.CSVMetaData; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.epgm.GraphCollection; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.cypher.capf.TestData; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.CAPFQueryResult; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.*; + +import static junit.framework.TestCase.fail; +import static org.gradoop.flink.model.impl.operators.cypher.capf.TestData.*; + +@RunWith(Parameterized.class) +public class CAPFPatternMatchingTest extends GradoopFlinkTestBase { + + protected final String testName; + + protected final String dataGraph; + + protected final String queryGraph; + + protected final String[] expectedGraphVariables; + + protected final String expectedCollection; + + private final MetaData metaData; + + public CAPFPatternMatchingTest(String testName, String dataGraph, String queryGraph, + String expectedGraphVariables, String expectedCollection) { + this.testName = testName; + this.dataGraph = dataGraph; + this.queryGraph = queryGraph; + this.expectedGraphVariables = expectedGraphVariables.split(","); + this.expectedCollection = expectedCollection; + + Map> vertexPropertyMap = new HashMap<>(); + Map> edgePropertyMap = new HashMap<>(); + + List propertyList = new ArrayList<>(); + propertyList.add(new PropertyMetaData("id", MetaData.TypeString.INTEGER.getTypeString(), + null)); + + vertexPropertyMap.put("A", propertyList); + vertexPropertyMap.put("B", propertyList); + vertexPropertyMap.put("C", propertyList); + vertexPropertyMap.put("D", propertyList); + + edgePropertyMap.put("a", propertyList); + edgePropertyMap.put("b", propertyList); + edgePropertyMap.put("c", propertyList); + edgePropertyMap.put("d", propertyList); + + metaData = new CSVMetaData(new HashMap<>(), vertexPropertyMap, edgePropertyMap); + } + + @Test + public void testGraphElementIdEquality() throws Exception { + FlinkAsciiGraphLoader loader = getLoaderFromString(dataGraph); + + // initialize with data graph + LogicalGraph db = loader.getLogicalGraphByVariable(TestData.DATA_GRAPH_VARIABLE); + + // append the expected result + loader.appendToDatabaseFromString(expectedCollection); + + // execute and validate + CAPFQueryResult capfResult = db.cypher(queryGraph, metaData); + if (capfResult.containsGraphs()) { + GraphCollection result = capfResult.getGraphs(); + GraphCollection expected = loader.getGraphCollectionByVariables(expectedGraphVariables); + collectAndAssertTrue(result.equalsByGraphElementIds(expected)); + } else { + fail("Result did not contain any graphs!"); + } + } + + @Test + public void testGraphElementEquality() throws Exception { + FlinkAsciiGraphLoader loader = getLoaderFromString(dataGraph); + + // initialize with data graph + LogicalGraph db = loader.getLogicalGraphByVariable(TestData.DATA_GRAPH_VARIABLE); + + // append the expected result + loader.appendToDatabaseFromString(expectedCollection); + + // execute and validate + CAPFQueryResult capfResult = db.cypher(queryGraph, metaData); + if (capfResult.containsGraphs()) { + GraphCollection result = capfResult.getGraphs(); + GraphCollection expected = loader.getGraphCollectionByVariables(expectedGraphVariables); + collectAndAssertTrue(result.equalsByGraphElementData(expected)); + } else { + fail("Result did not contain any graphs!"); + } + } + + @Parameterized.Parameters(name = "{index}: {0}") + public static Iterable data() { + return Arrays.asList( + new String[] { + "Graph1_Chain0", + GRAPH_1, CHAIN_PATTERN_0, + "expected1,expected2,expected3", + "expected1[(v1)-[e2]->(v6)]" + + "expected2[(v2)-[e3]->(v6)]" + + "expected3[(v5)-[e6]->(v4)]" + }, + new String[] { + "Graph1_Chain2", + GRAPH_1, CHAIN_PATTERN_2, + "expected1,expected2,expected3", + "expected1[(v1)]" + + "expected2[(v2)]" + + "expected3[(v5)]" + }, + new String[] { + "Graph2_Chain3", + GRAPH_2, CHAIN_PATTERN_3, + "expected1", + "expected1[(v9)-[e15]->(v9)]" + }, + new String[] { + "Graph2_Loop0", + GRAPH_2, + LOOP_PATTERN_0, + "expected1", + "expected1[(v9)-[e15]->(v9)]" + }, + new String[] { + "Graph1_Cycle2", + GRAPH_1, + CYCLE_PATTERN_2, + "expected1", + "expected1[" + + "(v2)-[e3]->(v6)" + + "(v6)-[e7]->(v2)" + + "(v6)-[e9]->(v7)" + + "]" + }, + new String[] { + "Graph1_Cycle4", + GRAPH_1, + CYCLE_PATTERN_4, + "expected1,expected2", + "expected1[(v1)-[e2]->(v6)-[e8]->(v5)-[e6]->(v4)-[e4]->(v1)]" + + "expected2[(v5)-[e6]->(v4)-[e4]->(v1)-[e2]->(v6)-[e8]->(v5)]" + }, + new String[] { + "Graph2_Cycle5", + GRAPH_2, + CYCLE_PATTERN_5, + "expected1,expected2", + "expected1[(v0)-[e1]->(v4)<-[e2]-(v0)]" + + "expected2[(v5)-[e9]->(v4)<-[e10]-(v5)]" + }, + new String[] { + "Graph4_Cycle6", + GRAPH_4, CYCLE_PATTERN_6, + "expected1,expected2", + "expected1[(v1)-[e1]->(v2)<-[e2]-(v1)]" + + "expected2[(v1)-[e2]->(v2)<-[e1]-(v1)]" + }, + new String[] { + "Graph4_Chain4", + GRAPH_4, CHAIN_PATTERN_4, + "expected1,expected2,expected3,expected4", + "expected1[(v0)-[e0]->(v1)-[e1]->(v2)]" + + "expected2[(v0)-[e0]->(v1)-[e2]->(v2)]" + + "expected3[(v1)-[e1]->(v2)-[e3]->(v3)]" + + "expected4[(v1)-[e2]->(v2)-[e3]->(v3)]" + }, + new String[] { + "Graph5_Chain6", + GRAPH_5, + CHAIN_PATTERN_6, + "expected1", + "expected1[ ]" + }, + new String[] { + "Graph6_Chain7", + GRAPH_5, + CHAIN_PATTERN_7, + "expected1", + "expected1[ ]" + }, + new String[] { + "Graph3_Unlabeled0", + GRAPH_3, + UNLABELED_PATTERN_0, + "expected1,expected2,expected3,expected4", + "expected1[(v0)]" + + "expected2[(v1)]" + + "expected3[(v2)]" + + "expected4[(v3)]" + }, + new String[] { + "Graph3_Unlabeled1", + GRAPH_3, + UNLABELED_PATTERN_1, + "expected1,expected2,expected3", + "expected1[(v0),(v1)]" + + "expected2[(v1),(v2)]" + + "expected3[(v2),(v3)]" + }, + new String[] { + "Graph1_Unlabeled2", + GRAPH_1, + UNLABELED_PATTERN_2, + "expected1,expected2", + "expected1[(v4)-[e5]->(v3)]" + + "expected2[(v6)-[e9]->(v7)]" + }, + new String[] { + "Graph1_Unlabeled3", + GRAPH_1, + UNLABELED_PATTERN_3, + "expected1,expected2,expected3", + "expected1[(v1),(v6)]" + + "expected2[(v5),(v4)]" + + "expected3[(v2),(v6)]" + } + ); + } + + private void printTable(Table table) { + table.writeToSink(new PrintTableSink()); + } + + private class PrintTableSink implements BatchTableSink { + + private String[] fieldNames; + + private TypeInformation[] fieldTypes; + + public TypeInformation getOutputType() { + return new RowTypeInfo(fieldTypes); + } + + public String[] getFieldNames() { + return fieldNames; + } + + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + public PrintTableSink configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + @Override + public void emitDataSet(DataSet dataSet) { + try { + dataSet.print(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQueryTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQueryTest.java new file mode 100644 index 000000000000..71731d26746b --- /dev/null +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/cypher/capf/query/CAPFQueryTest.java @@ -0,0 +1,248 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.cypher.capf.query; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.table.api.scala.BatchTableEnvironment; +import org.apache.flink.types.Row; +import org.gradoop.common.model.impl.metadata.MetaData; +import org.gradoop.common.model.impl.metadata.PropertyMetaData; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.flink.io.impl.csv.metadata.CSVMetaData; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.epgm.GraphCollection; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.cypher.capf.TestData; +import org.gradoop.flink.model.impl.operators.cypher.capf.result.CAPFQueryResult; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static junit.framework.TestCase.assertEquals; + +public class CAPFQueryTest extends GradoopFlinkTestBase { + + @Test + public void testCAPFProjection() throws Exception { + FlinkAsciiGraphLoader loader = getLoaderFromString(TestData.GRAPH_1); + + loader.appendToDatabaseFromString( + "expected1[(v1)], expected2[(v3)], expected3[(v5)], expected4[(v6)]" + + "expected5[(v1)], expected6[(v3)], expected7[(v5)], expected8[(v6)]"); + + LogicalGraph graph = loader.getLogicalGraphByVariable(TestData.DATA_GRAPH_VARIABLE); + + Map> vertexPropertyMap = new HashMap<>(); + Map> edgePropertyMap = new HashMap<>(); + + List propertyList = new ArrayList<>(); + propertyList.add(new PropertyMetaData("id", MetaData.TypeString.INTEGER.getTypeString(), + null)); + + vertexPropertyMap.put("A", propertyList); + vertexPropertyMap.put("B", propertyList); + vertexPropertyMap.put("C", propertyList); + vertexPropertyMap.put("D", propertyList); + + edgePropertyMap.put("a", propertyList); + edgePropertyMap.put("b", propertyList); + edgePropertyMap.put("c", propertyList); + + MetaData metaData = new CSVMetaData(new HashMap<>(), vertexPropertyMap, edgePropertyMap); + + CAPFQueryResult result = graph.cypher("MATCH (n1)-->(n2)<--(n3) RETURN n2", metaData); + + // because the pattern is symmetric, each result exists twice + GraphCollection expectedGraphs = loader.getGraphCollectionByVariables( + "expected1", "expected2", "expected3", "expected4", + "expected5", "expected6", "expected7", "expected8"); + + // execute and validate + GraphCollection resultGraphs = result.getGraphs(); + collectAndAssertTrue(resultGraphs.equalsByGraphElementIds(expectedGraphs)); + } + + @Test + public void testCAPFProjectionWithoutPropertyMaps() throws Exception { + FlinkAsciiGraphLoader loader = getLoaderFromString(TestData.GRAPH_1); + + loader.appendToDatabaseFromString( + "expected1[(v1)], expected2[(v3)], expected3[(v5)], expected4[(v6)]" + + "expected5[(v1)], expected6[(v3)], expected7[(v5)], expected8[(v6)]"); + + LogicalGraph graph = loader.getLogicalGraphByVariable(TestData.DATA_GRAPH_VARIABLE); + + CAPFQuery op = new CAPFQuery( + "MATCH (n1)-->(n2)<--(n3) RETURN n2", + this.getExecutionEnvironment() + ); + + CAPFQueryResult result = op.execute(graph); + + // because the pattern is symmetric, each result exists twice + GraphCollection expectedGraphs = loader.getGraphCollectionByVariables( + "expected1", "expected2", "expected3", "expected4", + "expected5", "expected6", "expected7", "expected8"); + + // execute and validate + GraphCollection resultGraphs = result.getGraphs(); + collectAndAssertTrue(resultGraphs.equalsByGraphElementIds(expectedGraphs)); + } + + @Test + public void testCAPFWithByteArrayPayload() throws Exception { + FlinkAsciiGraphLoader loader = getLoaderFromString(TestData.GRAPH_1); + + loader.appendToDatabaseFromString( + "expected1[(v1)], expected2[(v3)], expected3[(v5)], expected4[(v6)]" + + "expected5[(v1)], expected6[(v3)], expected7[(v5)], expected8[(v6)]"); + + LogicalGraph graph = loader.getLogicalGraphByVariable(TestData.DATA_GRAPH_VARIABLE); + + DataSet verticesWithPayload = graph.getVertices() + .map((MapFunction) vertex -> { + vertex.setProperty("map", new HashMap()); + return vertex; + }); + + LogicalGraph graphWithPayload = graph.getFactory() + .fromDataSets(verticesWithPayload, graph.getEdges()); + + CAPFQuery op = new CAPFQuery( + "MATCH (n1)-->(n2)<--(n3) RETURN n2", + this.getExecutionEnvironment() + ); + + CAPFQueryResult result = op.execute(graphWithPayload); + + // because the pattern is symmetric, each result exists twice + GraphCollection expectedGraphs = loader.getGraphCollectionByVariables( + "expected1", "expected2", "expected3", "expected4", + "expected5", "expected6", "expected7", "expected8"); + + // execute and validate + GraphCollection resultGraphs = result.getGraphs(); + collectAndAssertTrue(resultGraphs.equalsByGraphElementIds(expectedGraphs)); + } + + @Test + public void testCAPFProperties() throws Exception { + FlinkAsciiGraphLoader loader = getLoaderFromString(TestData.GRAPH_1); + + LogicalGraph graph = loader.getLogicalGraphByVariable(TestData.DATA_GRAPH_VARIABLE); + + Map> vertexPropertyMap = new HashMap<>(); + Map> edgePropertyMap = new HashMap<>(); + + List propertyList = new ArrayList<>(); + propertyList.add(new PropertyMetaData("id", MetaData.TypeString.INTEGER.getTypeString(), + null)); + + vertexPropertyMap.put("A", propertyList); + vertexPropertyMap.put("B", propertyList); + vertexPropertyMap.put("C", propertyList); + vertexPropertyMap.put("D", propertyList); + + edgePropertyMap.put("a", propertyList); + edgePropertyMap.put("b", propertyList); + edgePropertyMap.put("c", propertyList); + + MetaData metaData = new CSVMetaData(new HashMap<>(), vertexPropertyMap, edgePropertyMap); + + CAPFQueryResult result = graph.cypher( + "MATCH (n1)-->(n2)-->(n3) RETURN n1.id, n2.id, n3.id", + metaData); + + BatchTableEnvironment tenv = (BatchTableEnvironment) result.getTable().tableEnv(); + DataSet resultDataSet = + tenv.toDataSet(result.getTable(), TypeInformation.of(Row.class)).javaSet(); + + Long[][] expectedIds = { + {0L, 1L, 1L, 1L, 2L, 2L, 2L, 4L, 5L, 5L, 5L, 6L, 6L, 6L, 8L, 8L}, + {1L, 6L, 6L, 6L, 6L, 6L, 6L, 1L, 4L, 4L, 9L, 2L, 5L, 5L, 5L, 5L}, + {6L, 2L, 5L, 7L, 2L, 5L, 7L, 6L, 1L, 3L, 10L, 6L, 4L, 9L, 4L, 9L} + }; + + List resultList = resultDataSet.collect(); + + assertEquals(expectedIds[0].length, resultList.size()); + + for (Row r : resultList) { + assertEquals(3, r.getArity()); + } + + resultList.sort((r1, r2) -> { + for (int i = 0; i < r1.getArity(); i++) { + int comp = ((Long) r1.getField(i)).compareTo((Long) r2.getField(i)); + if (comp != 0) { + return comp; + } + } + return 0; + }); + + for (int i = 0; i < expectedIds.length; i++) { + assertEquals(expectedIds[0][i], resultList.get(i).getField(0)); + assertEquals(expectedIds[1][i], resultList.get(i).getField(1)); + assertEquals(expectedIds[2][i], resultList.get(i).getField(2)); + } + } + + @Test + public void testCAPFAggregation() throws Exception { + FlinkAsciiGraphLoader loader = getLoaderFromString(TestData.GRAPH_2); + + LogicalGraph graph = loader.getLogicalGraphByVariable(TestData.DATA_GRAPH_VARIABLE); + + Map> vertexPropertyMap = new HashMap<>(); + Map> edgePropertyMap = new HashMap<>(); + + List propertyList = new ArrayList<>(); + propertyList.add(new PropertyMetaData("id", MetaData.TypeString.INTEGER.getTypeString(), + null)); + + vertexPropertyMap.put("A", propertyList); + vertexPropertyMap.put("B", propertyList); + vertexPropertyMap.put("C", propertyList); + vertexPropertyMap.put("D", propertyList); + + edgePropertyMap.put("a", propertyList); + edgePropertyMap.put("b", propertyList); + edgePropertyMap.put("c", propertyList); + edgePropertyMap.put("d", propertyList); + + MetaData metaData = new CSVMetaData(new HashMap<>(), vertexPropertyMap, edgePropertyMap); + + CAPFQueryResult result = graph.cypher( + "MATCH (n1) RETURN avg(n1.id)", + metaData + ); + + BatchTableEnvironment tenv = (BatchTableEnvironment) result.getTable().tableEnv(); + DataSet resultDataSet = tenv.toDataSet(result.getTable(), TypeInformation.of(Row.class)).javaSet(); + + List resultList = resultDataSet.collect(); + assertEquals(1, resultList.size()); + assertEquals(1, resultList.get(0).getArity()); + assertEquals(6L, (long) resultList.get(0).getField(0)); + } +} diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/common/statistics/GraphStatisticsHDFSReaderTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/common/statistics/GraphStatisticsHDFSReaderTest.java index 60f4e7ab4218..86005e248a71 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/common/statistics/GraphStatisticsHDFSReaderTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/common/statistics/GraphStatisticsHDFSReaderTest.java @@ -56,7 +56,7 @@ public static void setUp() throws Exception { /** * Stops the test cluster after the test. * - * @throws Exception + * @throws Exception on failure */ @AfterClass public static void tearDown() throws Exception { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/QueryEngineITTests.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/QueryEngineITTests.java index 6f086b16d84a..a514d7725f3e 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/QueryEngineITTests.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/QueryEngineITTests.java @@ -82,7 +82,7 @@ public void testMatchWithValueJoin() throws Exception { * @param q cypher query * @param estimatedCardinality estimated cardinality of the result * @param exactCardinality exact cardinality of the result - * @throws Exception + * @throws Exception on failure */ private void assertCardinalities(String q, long estimatedCardinality, int exactCardinality) throws Exception { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/common/pojos/EmbeddingTestUtils.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/common/pojos/EmbeddingTestUtils.java index 42d200752137..4ffbb429489a 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/common/pojos/EmbeddingTestUtils.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/common/pojos/EmbeddingTestUtils.java @@ -57,7 +57,7 @@ public static void assertEmbedding(Embedding embedding, * * @param embeddings data set containing embedding * @param path expected path - * @throws Exception + * @throws Exception on failure */ public static void assertEmbeddingExists(DataSet embeddings, GradoopId... path) throws Exception { @@ -72,7 +72,7 @@ public static void assertEmbeddingExists(DataSet embeddings, GradoopI * * @param embeddings data set containing embeddings * @param predicate predicate - * @throws Exception + * @throws Exception on failure */ public static void assertEmbeddingExists(DataSet embeddings, Predicate predicate) throws Exception { @@ -94,7 +94,7 @@ public static void assertEmbeddingExists(List embeddings, Predicate dataSet, Consumer consumer) throws Exception { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/expand/functions/CreateInitialExpandEmbeddingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/expand/functions/CreateInitialExpandEmbeddingTest.java index da217720c212..97a7ecff8f17 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/expand/functions/CreateInitialExpandEmbeddingTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/expand/functions/CreateInitialExpandEmbeddingTest.java @@ -116,7 +116,7 @@ public void testEdgeIsomorphismWithLoop() throws Exception { * @param distinctEdges distinct edge columns of the base embedding * @param closingColumn closing column * @param isResult if true it is expected that the join yields exactly one result, 0 otherwise - * @throws Exception + * @throws Exception on failure */ private void testJoin(Embedding edge, List distinctVertices, List distinctEdges, int closingColumn, boolean isResult) throws Exception { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/join/functions/MergeEmbeddingsTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/join/functions/MergeEmbeddingsTest.java index c02fe2806556..0faf391bc69c 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/join/functions/MergeEmbeddingsTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/matching/single/cypher/operators/join/functions/MergeEmbeddingsTest.java @@ -495,7 +495,7 @@ public void testVertexIsomorphismEdgeIsomorphism() throws Exception { * @param distinctVertexColumnsLeft join operator argument * @param distinctEdgeColumnsLeft join operator argument * @param expectedEmbedding expected result - * @throws Exception + * @throws Exception on failure */ private void testMorphisms(List distinctVertexColumnsLeft, List distinctVertexColumnsRight, List distinctEdgeColumnsLeft, List distinctEdgeColumnsRight, List expectedEmbedding) throws Exception { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/PageRankSamplingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/PageRankSamplingTest.java index d9d6495a876a..ad6a6ac1ac9e 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/PageRankSamplingTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/PageRankSamplingTest.java @@ -21,6 +21,7 @@ import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.common.model.impl.pojo.Vertex; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingConstants; import org.junit.runners.Parameterized; import java.util.Arrays; @@ -70,17 +71,17 @@ public void validateSpecific(LogicalGraph input, LogicalGraph output) { try { // test normal graph GraphHead normalGh = output.getGraphHead().collect().get(0); - double minScore = normalGh.getPropertyValue(PageRankSampling.MIN_PAGE_RANK_SCORE_PROPERTY_KEY) + double minScore = normalGh.getPropertyValue(SamplingConstants.MIN_PAGE_RANK_SCORE_PROPERTY_KEY) .getDouble(); - double maxScore = normalGh.getPropertyValue(PageRankSampling.MAX_PAGE_RANK_SCORE_PROPERTY_KEY) + double maxScore = normalGh.getPropertyValue(SamplingConstants.MAX_PAGE_RANK_SCORE_PROPERTY_KEY) .getDouble(); if (minScore != maxScore) { for (Vertex v : newVertices) { assertTrue("vertex does not have scaled PageRank-score property (should have):" + - v.toString(), v.hasProperty(PageRankSampling.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY)); + v.toString(), v.hasProperty(SamplingConstants.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY)); - if (v.hasProperty(PageRankSampling.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY)) { - double score = v.getPropertyValue(PageRankSampling.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY) + if (v.hasProperty(SamplingConstants.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY)) { + double score = v.getPropertyValue(SamplingConstants.SCALED_PAGE_RANK_SCORE_PROPERTY_KEY) .getDouble(); if (sampleGreaterThanThreshold) { assertTrue("sampled vertex has PageRankScore smaller or equal than threshold", @@ -129,9 +130,9 @@ public void validateSpecific(LogicalGraph input, LogicalGraph output) { GraphHead specialGh = sampledGraph.getGraphHead().collect().get(0); double minScore1 = specialGh.getPropertyValue( - PageRankSampling.MIN_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); + SamplingConstants.MIN_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); double maxScore1 = specialGh.getPropertyValue( - PageRankSampling.MAX_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); + SamplingConstants.MAX_PAGE_RANK_SCORE_PROPERTY_KEY).getDouble(); assertEquals("min PageRankScore is not equal to max PageRankScore", minScore1, maxScore1, 0.0); diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/ParameterizedTestForGraphSampling.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/ParameterizedTestForGraphSampling.java index 11f2eb534eef..47fa28f53068 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/ParameterizedTestForGraphSampling.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/ParameterizedTestForGraphSampling.java @@ -69,6 +69,10 @@ public abstract class ParameterizedTestForGraphSampling extends GradoopFlinkTest * The iteration number used by Flinks PageRank-algorithm */ int maxIteration = 20; + /** + * Number of start vertices used by RandomWalkSampling + */ + int numberOfStartVertices = 5; /** * Whether to sample vertices with PageRank-score greater (true) or equal/smaller (false) the * sampleSize @@ -121,7 +125,7 @@ public abstract class ParameterizedTestForGraphSampling extends GradoopFlinkTest * @param seed Seed-value for random number generator, e.g. 0 * @param sampleSize Value for sample size, e.g. 0.5 */ - public ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize) { + ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize) { this.testName = testName; this.seed = seed; this.sampleSize = sampleSize; @@ -135,7 +139,7 @@ public ParameterizedTestForGraphSampling(String testName, long seed, float sampl * @param sampleSize Value for sample size, e.g. 0.5 * @param neighborType The vertex neighborhood type, e.g. Neighborhood.BOTH */ - public ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize, + ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize, Neighborhood neighborType) { this(testName, seed, sampleSize); this.neighborType = neighborType; @@ -150,7 +154,7 @@ public ParameterizedTestForGraphSampling(String testName, long seed, float sampl * @param dampeningFactor The dampening factor used by Flinks PageRank-algorithm, e.g. 0.85 * @param maxIteration The iteration number used by Flinks PageRank-algorithm, e.g. 20 */ - public ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize, + ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize, double dampeningFactor, int maxIteration, boolean sampleGreaterThanThreshold, boolean keepVerticesIfSameScore) { this(testName, seed, sampleSize); @@ -169,7 +173,7 @@ public ParameterizedTestForGraphSampling(String testName, long seed, float sampl * @param degreeType The vertex degree type, e.g. VertexDegree.BOTH * @param degreeThreshold The threshold for the vertex degree, e.g. 3 */ - public ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize, + ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize, VertexDegree degreeType, long degreeThreshold) { this(testName, seed, sampleSize); this.degreeType = degreeType; @@ -185,7 +189,7 @@ public ParameterizedTestForGraphSampling(String testName, long seed, float sampl * @param edgeSampleSize Value for edge sample size, e.g. 0.5 * @param vertexEdgeSamplingType Type for VertexEdgeSampling, e.g. SimpleVersion */ - public ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize, + ParameterizedTestForGraphSampling(String testName, long seed, float sampleSize, float edgeSampleSize, RandomVertexEdgeSampling.VertexEdgeSamplingType vertexEdgeSamplingType) { this(testName, seed, sampleSize); this.edgeSampleSize = edgeSampleSize; diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomEdgeSamplingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomEdgeSamplingTest.java index a2e4189c1305..8d4b78d18219 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomEdgeSamplingTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomEdgeSamplingTest.java @@ -39,17 +39,11 @@ public RandomEdgeSamplingTest(String testName, String seed, String sampleSize) { super(testName, Long.parseLong(seed), Float.parseFloat(sampleSize)); } - /** - * {@inheritDoc} - */ @Override public SamplingAlgorithm getSamplingOperator() { return new RandomEdgeSampling(sampleSize, seed); } - /** - * {@inheritDoc} - */ @Override public void validateSpecific(LogicalGraph input, LogicalGraph output) { Set connectedVerticesIDs = new HashSet<>(); diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomLimitedDegreeVertexSamplingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomLimitedDegreeVertexSamplingTest.java index 3736da957422..43195c6948c5 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomLimitedDegreeVertexSamplingTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomLimitedDegreeVertexSamplingTest.java @@ -46,17 +46,11 @@ public RandomLimitedDegreeVertexSamplingTest(String testName, String seed, Strin VertexDegree.valueOf(degreeType), Long.parseLong(degreeThreshold)); } - /** - * {@inheritDoc} - */ @Override public SamplingAlgorithm getSamplingOperator() { return new RandomLimitedDegreeVertexSampling(sampleSize, seed, degreeThreshold, degreeType); } - /** - * {@inheritDoc} - */ @Override public void validateSpecific(LogicalGraph input, LogicalGraph output) { List dbDegreeVertices = Lists.newArrayList(); diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomNonUniformVertexSamplingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomNonUniformVertexSamplingTest.java index 612287d28829..143b4b07228c 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomNonUniformVertexSamplingTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomNonUniformVertexSamplingTest.java @@ -36,17 +36,11 @@ public RandomNonUniformVertexSamplingTest(String testName, String seed, String s super(testName, Long.parseLong(seed), Float.parseFloat(sampleSize)); } - /** - * {@inheritDoc} - */ @Override public SamplingAlgorithm getSamplingOperator() { return new RandomNonUniformVertexSampling(sampleSize, seed); } - /** - * {@inheritDoc} - */ @Override public void validateSpecific(LogicalGraph input, LogicalGraph output) { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexEdgeSamplingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexEdgeSamplingTest.java index 43a406bda58f..ec861a512fd9 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexEdgeSamplingTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexEdgeSamplingTest.java @@ -37,17 +37,11 @@ public RandomVertexEdgeSamplingTest(String testName, String seed, String sampleS RandomVertexEdgeSampling.VertexEdgeSamplingType.valueOf(vertexEdgeSamplingType)); } - /** - * {@inheritDoc} - */ @Override public SamplingAlgorithm getSamplingOperator() { return new RandomVertexEdgeSampling(sampleSize, edgeSampleSize, seed, vertexEdgeSamplingType); } - /** - * {@inheritDoc} - */ @Override public void validateSpecific(LogicalGraph input, LogicalGraph output) { } diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexNeighborhoodSamplingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexNeighborhoodSamplingTest.java index 8ddc7bab12e4..330768568d20 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexNeighborhoodSamplingTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexNeighborhoodSamplingTest.java @@ -37,17 +37,11 @@ public RandomVertexNeighborhoodSamplingTest(String testName, String seed, String Neighborhood.valueOf(neighborType)); } - /** - * {@inheritDoc} - */ @Override public SamplingAlgorithm getSamplingOperator() { return new RandomVertexNeighborhoodSampling(sampleSize, seed, neighborType); } - /** - * {@inheritDoc} - */ @Override public void validateSpecific(LogicalGraph input, LogicalGraph output) { } diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexSamplingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexSamplingTest.java index e2534558ebf2..69dadd378a60 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexSamplingTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomVertexSamplingTest.java @@ -36,17 +36,11 @@ public RandomVertexSamplingTest(String testName, String seed, String sampleSize) super(testName, Long.parseLong(seed), Float.parseFloat(sampleSize)); } - /** - * {@inheritDoc} - */ @Override public SamplingAlgorithm getSamplingOperator() { return new RandomVertexSampling(sampleSize, seed); } - /** - * {@inheritDoc} - */ @Override public void validateSpecific(LogicalGraph input, LogicalGraph output) { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomWalkSamplingTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomWalkSamplingTest.java new file mode 100644 index 000000000000..58f5ae6412ee --- /dev/null +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/RandomWalkSamplingTest.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.sampling; + +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.junit.runners.Parameterized; + +import java.util.Arrays; + + +public class RandomWalkSamplingTest extends ParameterizedTestForGraphSampling { + + public RandomWalkSamplingTest(String testName, String seed, String sampleSize) { + super(testName, Long.parseLong(seed), Float.parseFloat(sampleSize)); + } + + @Override + public SamplingAlgorithm getSamplingOperator() { + return new RandomWalkSampling(sampleSize, numberOfStartVertices); + } + + @Override + public void validateSpecific(LogicalGraph input, LogicalGraph output) { + } + + /** + * Parameters called when running the test + * + * @return List of parameters + */ + @Parameterized.Parameters(name = "{index}: {0}") + public static Iterable data() { + return Arrays.asList(new String[] { + "RandomWalkSamplingTest with seed", + "-4181668494294894490", + "0.272f" + }, new String[] { + "RandomWalkSamplingTest without seed", + "0", + "0.272f" + }); + } +} diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageClusteringCoefficientTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageClusteringCoefficientTest.java index b76dc66aa954..24b9e14aa367 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageClusteringCoefficientTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageClusteringCoefficientTest.java @@ -18,6 +18,7 @@ import org.gradoop.common.model.impl.pojo.GraphHead; import org.gradoop.flink.model.GradoopFlinkTestBase; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.AverageClusteringCoefficient; import org.gradoop.flink.util.FlinkAsciiGraphLoader; import org.junit.Before; import org.junit.Test; diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageDegreeTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageDegreeTest.java index c325fd70a466..fd9dc8d87afa 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageDegreeTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageDegreeTest.java @@ -17,6 +17,8 @@ import org.gradoop.flink.model.GradoopFlinkTestBase; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.AverageDegree; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageIncomingDegreeTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageIncomingDegreeTest.java index 0910a4eea902..97faaee8fb1e 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageIncomingDegreeTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageIncomingDegreeTest.java @@ -17,6 +17,8 @@ import org.gradoop.flink.model.GradoopFlinkTestBase; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.AverageIncomingDegree; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageOutgoingDegreeTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageOutgoingDegreeTest.java index b904dcc0033f..887f483d73bf 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageOutgoingDegreeTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/AverageOutgoingDegreeTest.java @@ -17,6 +17,8 @@ import org.gradoop.flink.model.GradoopFlinkTestBase; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.AverageOutgoingDegree; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ConnectedComponentsDistributionTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ConnectedComponentsDistributionTest.java index d38b804ca968..4c19e8a71715 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ConnectedComponentsDistributionTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ConnectedComponentsDistributionTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.java.tuple.Tuple3; import org.gradoop.flink.model.GradoopFlinkTestBase; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.ConnectedComponentsDistribution; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/DegreeCentralityTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/DegreeCentralityTest.java new file mode 100644 index 000000000000..49f3b2eb9f93 --- /dev/null +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/DegreeCentralityTest.java @@ -0,0 +1,120 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.sampling.statistics; + +import org.apache.flink.api.java.DataSet; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.DegreeCentrality; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Test class for degree centrality + */ +public class DegreeCentralityTest extends GradoopFlinkTestBase { + + /** + * graph as String for testing + * g0: a star graph + * g1: a path + * g2: a community graph + */ + private String graphString = + "g0[" + + "(v0)-[]->(v1)" + + "(v0)-[]->(v2)" + + "(v0)-[]->(v3)" + + "(v0)-[]->(v4)" + + "(v0)-[]->(v5)" + + "]" + + "g1[" + + "(v0)-[]->(v1)" + + "(v1)-[]->(v2)" + + "(v2)-[]->(v3)" + + "(v3)-[]->(v4)" + + "]" + + "g2[" + + "(v0)-[]->(v1)" + + "(v1)-[]->(v2)" + + "(v1)-[]->(v3)" + + "(v2)-[]->(v3)" + + "(v0)-[]->(v4)" + + "(v4)-[]->(v5)" + + "(v4)-[]->(v6)" + + "(v5)-[]->(v6)" + + "]" + + "g3[" + + "(v0)-[]->(v1)" + + "(v0)-[]->(v2)" + + "(v0)-[]->(v3)" + + "(v0)-[]->(v4)" + + "(v0)-[]->(v5)" + + "(v0)-[]->(v6)" + + "(v6)-[]->(v7)" + + "(v6)-[]->(v8)" + + "(v6)-[]->(v9)" + + "]"; + + /** + * Test star graph for degree centrality + * + * @throws Exception throws any Exception + */ + @Test + public void testStar() throws Exception { + LogicalGraph graph = getLoaderFromString(graphString).getLogicalGraphByVariable("g0"); + DataSet dataSet = new DegreeCentrality().execute(graph); + assertEquals(1.0, dataSet.collect().get(0), 0.001); + } + + /** + * Test path graph for degree centrality + * + * @throws Exception throws any Exception + */ + @Test + public void testPath() throws Exception { + LogicalGraph graph = getLoaderFromString(graphString).getLogicalGraphByVariable("g1"); + DataSet dataSet = new DegreeCentrality().execute(graph); + assertEquals(0.167, dataSet.collect().get(0), 0.001); + } + + /** + * Test community graph for degree centrality + * + * @throws Exception throws any Exception + */ + @Test + public void testCommunity() throws Exception { + LogicalGraph graph = getLoaderFromString(graphString).getLogicalGraphByVariable("g2"); + DataSet dataSet = new DegreeCentrality().execute(graph); + assertEquals(0.167, dataSet.collect().get(0), 0.001); + } + + /** + * Test connected stars graph for degree centrality + * + * @throws Exception throws any Exception + */ + @Test + public void testConnectedStars() throws Exception { + LogicalGraph graph = getLoaderFromString(graphString).getLogicalGraphByVariable("g3"); + DataSet dataSet = new DegreeCentrality().execute(graph); + assertEquals(0.583, dataSet.collect().get(0), 0.001); + } +} diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/GraphDensityTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/GraphDensityTest.java index c4437a3bb343..d14db331245d 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/GraphDensityTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/GraphDensityTest.java @@ -17,6 +17,8 @@ import org.gradoop.flink.model.GradoopFlinkTestBase; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.GraphDensity; +import org.gradoop.flink.model.impl.operators.sampling.common.SamplingEvaluationConstants; import org.junit.Test; import static org.junit.Assert.assertTrue; diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ValueConnectedComponentsTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ValueConnectedComponentsTest.java index 4e7e1fb8fc07..bb1fe63438e4 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ValueConnectedComponentsTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/sampling/statistics/ValueConnectedComponentsTest.java @@ -20,6 +20,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.gradoop.flink.model.GradoopFlinkTestBase; import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.operators.statistics.ConnectedComponentsDistributionAsValues; import org.gradoop.flink.util.FlinkAsciiGraphLoader; import org.junit.Test; @@ -84,7 +85,7 @@ public void testValueConnectedComponents() throws Exception { // execute Gelly ConnectedComponents. DataSet> cComponents = - new ValueConnectedComponentsDistribution(Integer.MAX_VALUE).execute(graph); + new ConnectedComponentsDistributionAsValues(Integer.MAX_VALUE).execute(graph); List> vertexComponentList = new ArrayList<>(); cComponents.output(new LocalCollectionOutputFormat<>(vertexComponentList)); diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/subgraph/SubgraphTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/subgraph/SubgraphTest.java index 383baa4af3c4..0a6ad8edc632 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/subgraph/SubgraphTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/subgraph/SubgraphTest.java @@ -106,7 +106,7 @@ public void testPartialSubgraph() throws Exception { /** * Extracts a subgraph which is empty. * - * @throws Exception + * @throws Exception on failure */ @Test public void testEmptySubgraph() throws Exception { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/transformation/TransformationTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/transformation/TransformationTest.java index 95794458e9bb..c92e40184c0a 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/transformation/TransformationTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/transformation/TransformationTest.java @@ -124,7 +124,7 @@ public void testIdEquality() throws Exception { /** * Tests the data in the resulting graph. * - * @throws Exception + * @throws Exception on failure */ @Test public void testDataEquality() throws Exception { diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/verify/VerifyTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/verify/VerifyTest.java new file mode 100644 index 000000000000..2a9d74740970 --- /dev/null +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/operators/verify/VerifyTest.java @@ -0,0 +1,95 @@ +/* + * Copyright © 2014 - 2019 Leipzig University (Database Research Group) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradoop.flink.model.impl.operators.verify; + +import org.apache.flink.api.java.io.LocalCollectionOutputFormat; +import org.gradoop.common.model.impl.id.GradoopId; +import org.gradoop.common.model.impl.pojo.Edge; +import org.gradoop.common.model.impl.pojo.Vertex; +import org.gradoop.common.model.impl.properties.PropertyValue; +import org.gradoop.flink.model.GradoopFlinkTestBase; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; +import org.gradoop.flink.model.impl.functions.bool.True; +import org.gradoop.flink.model.impl.functions.epgm.ByProperty; +import org.gradoop.flink.model.impl.operators.subgraph.Subgraph; +import org.gradoop.flink.util.FlinkAsciiGraphLoader; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +/** + * Test for the {@link Verify} operator. + */ +public class VerifyTest extends GradoopFlinkTestBase { + + /** + * Test the verify operator with a subgraph operator. + * + * @throws Exception when the execution in Flink fails. + */ + @Test + public void testWithSubgraph() throws Exception { + FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); + loader.appendToDatabaseFromString("expected[" + + "(eve)-[ekb:knows {since : 2015}]->(bob)" + + "]"); + LogicalGraph input = loader.getLogicalGraphByVariable("g0"); + // Apply a subgraph operator that would result in dangling edges. + LogicalGraph subgraph = input.subgraph( + new ByProperty("name", PropertyValue.create("Alice")).negate(), + new True<>(), Subgraph.Strategy.BOTH); + // Make sure that the graph contains dangling edges. + List danglingEdges = getDanglingEdges(subgraph); + List expectedDanglingEdges = Arrays.asList(loader.getEdgeByVariable("eka"), + loader.getEdgeByVariable("akb"), loader.getEdgeByVariable("bka")); + Comparator comparator = Comparator.comparing(Edge::getId); + danglingEdges.sort(comparator); + expectedDanglingEdges.sort(comparator); + assertArrayEquals(expectedDanglingEdges.toArray(), danglingEdges.toArray()); + // Now run verify and check if those edges were removed. + LogicalGraph verifiedSubgraph = subgraph.verify(); + assertEquals("Verified graph contained dangling edges.", + 0, getDanglingEdges(verifiedSubgraph).size()); + // Check if nothing else has been removed (i.e. the result is correct) + collectAndAssertTrue(loader.getLogicalGraphByVariable("expected") + .equalsByElementData(verifiedSubgraph)); + } + + /** + * Determines all dangling edges of a graph. + * + * @throws Exception when the execution in Flink fails. + */ + private List getDanglingEdges(LogicalGraph graph) throws Exception { + List vertices = new ArrayList<>(); + List edges = new ArrayList<>(); + graph.getVertices().output(new LocalCollectionOutputFormat<>(vertices)); + graph.getEdges().output(new LocalCollectionOutputFormat<>(edges)); + getExecutionEnvironment().execute(); + Set ids = vertices.stream().map(Vertex::getId).collect(Collectors.toSet()); + return edges.stream() + .filter(e -> !ids.contains(e.getSourceId()) || !ids.contains(e.getTargetId())) + .collect(Collectors.toList()); + } +} diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/properties/PropertiesSerializationTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/properties/PropertiesSerializationTest.java index 99ac1dac09a3..f07641c53277 100644 --- a/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/properties/PropertiesSerializationTest.java +++ b/gradoop-flink/src/test/java/org/gradoop/flink/model/impl/properties/PropertiesSerializationTest.java @@ -24,6 +24,8 @@ import org.junit.Test; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -94,9 +96,41 @@ public void testLargePropertyList() throws Exception { .collect(Collectors.toList()); PropertyValue largeValue = PropertyValue.create(largeList); // Make sure the List was large enough. - assertTrue("PropertyValue to large enough.", + assertTrue("PropertyValue not large enough.", largeValue.byteSize() > PropertyValue.LARGE_PROPERTY_THRESHOLD); assertEquals("Property values were not equal.", largeValue, GradoopFlinkTestUtils.writeAndRead(largeValue, getExecutionEnvironment())); } + + @Test + public void testLargePropertyMap() throws Exception { + HashMap largeMap = new HashMap<>(); + long neededEntries = PropertyValue.LARGE_PROPERTY_THRESHOLD / 10; + for (int i = 0; i < neededEntries; i++) { + largeMap.put(PropertyValue.create("key" + i), PropertyValue.create("value" + i)); + } + + PropertyValue largeValue = PropertyValue.create(largeMap); + // Make sure the map was large enough. + assertTrue("PropertyValue not large enough.", + largeValue.byteSize() > PropertyValue.LARGE_PROPERTY_THRESHOLD); + assertEquals("Property values were not equal.", largeValue, + GradoopFlinkTestUtils.writeAndRead(largeValue, getExecutionEnvironment())); + } + + @Test + public void testLargePropertySet() throws Exception { + HashSet largeSet = new HashSet<>(); + long neededEntries = PropertyValue.LARGE_PROPERTY_THRESHOLD; + for (int i = 0; i < neededEntries; i++) { + largeSet.add(PropertyValue.create("entryNr" + i)); + } + + PropertyValue largeValue = PropertyValue.create(largeSet); + // Make sure the set was large enough. + assertTrue("PropertyValue not large enough.", + largeValue.byteSize() > PropertyValue.LARGE_PROPERTY_THRESHOLD); + assertEquals("Property values were not equal.", largeValue, + GradoopFlinkTestUtils.writeAndRead(largeValue, getExecutionEnvironment())); + } } diff --git a/gradoop-store/gradoop-accumulo/pom.xml b/gradoop-store/gradoop-accumulo/pom.xml index b1de45dda317..fe933f715449 100644 --- a/gradoop-store/gradoop-accumulo/pom.xml +++ b/gradoop-store/gradoop-accumulo/pom.xml @@ -5,7 +5,7 @@ gradoop-store org.gradoop - 0.4.4 + 0.4.5 4.0.0 @@ -105,12 +105,13 @@ - org.apache.maven.plugins maven-shade-plugin + + create-iterator-jar package shade @@ -126,8 +127,54 @@ com.esotericsoftware.kryo:* org.apache.flink:flink-core org.apache.hbase:hbase-common + org.objenesis:objenesis + + + + com.esotericsoftware + org.gradoop.thirdparty.com.esotericsoftware + + + org.objenesis + org.gradoop.thirdparty.org.objenesis + + + + + + + shade-jar + package + + shade + + + + + ${project.groupId}:${project.artifactId} + + com.esotericsoftware.minlog:* + com.esotericsoftware.kryo:* + org.objenesis:objenesis + + + + + + com.esotericsoftware + org.gradoop.thirdparty.com.esotericsoftware + + + org.objenesis + org.gradoop.thirdparty.org.objenesis + + + false + false @@ -160,7 +207,7 @@ com.esotericsoftware - kryo-shaded + kryo diff --git a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/AccumuloEPGMStore.java b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/AccumuloEPGMStore.java index 1204762a3de7..6f31dbe14928 100644 --- a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/AccumuloEPGMStore.java +++ b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/AccumuloEPGMStore.java @@ -227,6 +227,37 @@ public void close() { } } + /** + * Drop all tables used by this store instance. + * + * @throws IOException when deleting tables fails. + */ + public void dropTables() throws IOException { + for (String tableName : new String[] { + getVertexTableName(), getEdgeTableName(), getGraphHeadName()}) { + try { + dropTableIfExists(tableName); + } catch (AccumuloSecurityException | AccumuloException e) { + throw new IOException("Failed to delete table " + tableName, e); + } + } + } + + /** + * Truncate all tables handled by this store instance, i.e. delete all rows. + * The current implementations simply drops all tables and creates new ones. + * + * @throws IOException when deleting or creating tables fails. + */ + public void truncateTables() throws IOException { + dropTables(); + try { + createTablesIfNotExists(); + } catch (AccumuloSecurityException | AccumuloException e) { + throw new IOException("Failed to create tables.", e); + } + } + @Nullable @Override public GraphHead readGraph(@Nonnull GradoopId graphId) throws IOException { @@ -468,4 +499,22 @@ private void createTablesIfNotExists() throws AccumuloSecurityException, Accumul } } + /** + * Delete a table if it exists. + * + * @param tableName The table name. + * @throws AccumuloSecurityException if the user does not have persmissions to delete the table. + * @throws AccumuloException if a general Accumulo error occurs. + */ + private void dropTableIfExists(@Nonnull String tableName) throws AccumuloSecurityException, + AccumuloException { + if (conn.tableOperations().exists(tableName)) { + try { + conn.tableOperations().delete(tableName); + } catch (TableNotFoundException e) { + // We checked before if it exists, so this should not happen. + throw new IllegalStateException(e); + } + } + } } diff --git a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/io/AccumuloDataSink.java b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/io/AccumuloDataSink.java index 7b8e21805b0c..cc9b9aad0dc9 100644 --- a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/io/AccumuloDataSink.java +++ b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/io/AccumuloDataSink.java @@ -26,6 +26,7 @@ import org.gradoop.storage.impl.accumulo.io.outputformats.ElementOutputFormat; import javax.annotation.Nonnull; +import java.io.IOException; /** * Write graph or graph collection into accumulo store @@ -46,22 +47,25 @@ public AccumuloDataSink( } @Override - public void write(LogicalGraph logicalGraph) { + public void write(LogicalGraph logicalGraph) throws IOException { write(logicalGraph, false); } @Override - public void write(GraphCollection graphCollection) { + public void write(GraphCollection graphCollection) throws IOException { write(graphCollection, false); } @Override - public void write(LogicalGraph logicalGraph, boolean overwrite) { + public void write(LogicalGraph logicalGraph, boolean overwrite) throws IOException { write(getFlinkConfig().getGraphCollectionFactory().fromGraph(logicalGraph), overwrite); } @Override - public void write(GraphCollection graphCollection, boolean overWrite) { + public void write(GraphCollection graphCollection, boolean overWrite) throws IOException { + if (overWrite) { + getStore().truncateTables(); + } graphCollection.getGraphHeads() .output(new ElementOutputFormat<>(GraphHead.class, getAccumuloConfig())); graphCollection.getVertices() diff --git a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/io/inputformats/BaseInputFormat.java b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/io/inputformats/BaseInputFormat.java index 540a69ebfc25..ce659a2d6e5b 100644 --- a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/io/inputformats/BaseInputFormat.java +++ b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/io/inputformats/BaseInputFormat.java @@ -130,6 +130,7 @@ protected abstract void attachIterator( * * @param row accumulo result * @return gradoop element + * @throws IOException on failure */ protected abstract T mapRow(Map.Entry row) throws IOException; diff --git a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/BaseElementIterator.java b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/BaseElementIterator.java index 985717fa3a67..44e98102819f 100644 --- a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/BaseElementIterator.java +++ b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/BaseElementIterator.java @@ -73,6 +73,7 @@ public abstract class BaseElementIterator implements * * @param pair k-v pair from accumulo row * @return gradoop element instance + * @throws IOException on failure */ @Nonnull public abstract E fromRow(@Nonnull Map.Entry pair) throws IOException; @@ -82,6 +83,7 @@ public abstract class BaseElementIterator implements * * @param record gradoop element instance * @return k-v pair as accumulo row + * @throws IOException on failure */ @Nonnull public abstract Pair toRow(@Nonnull E record) throws IOException; @@ -92,6 +94,7 @@ public abstract class BaseElementIterator implements * * @param source origin store source * @return decoded epgm element + * @throws IOException on failure */ @Nullable public abstract E readLine( diff --git a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopEdgeIterator.java b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopEdgeIterator.java index 2a6dd89d9406..79375f3606f2 100644 --- a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopEdgeIterator.java +++ b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopEdgeIterator.java @@ -74,6 +74,7 @@ public Pair toRow(@Nonnull EPGMEdge record) throws IOException { * * @param source origin store source * @return decoded epgm element + * @throws IOException on failure */ @Nullable public Edge readLine( diff --git a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopGraphHeadIterator.java b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopGraphHeadIterator.java index 7dc35185176d..e3d3113dde0f 100644 --- a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopGraphHeadIterator.java +++ b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopGraphHeadIterator.java @@ -70,6 +70,7 @@ public Pair toRow(@Nonnull EPGMGraphHead record) throws IOException * * @param source origin store source * @return decoded epgm element + * @throws IOException on failure */ @Nullable public GraphHead readLine( diff --git a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopVertexIterator.java b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopVertexIterator.java index 5fd0b90b5a39..aa6d37e57b39 100644 --- a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopVertexIterator.java +++ b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/impl/accumulo/iterator/tserver/GradoopVertexIterator.java @@ -72,6 +72,7 @@ public Pair toRow(@Nonnull EPGMVertex record) throws IOException { * * @param source origin store source * @return decoded epgm element + * @throws IOException on failure */ @Nullable public Vertex readLine( diff --git a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/utils/KryoUtils.java b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/utils/KryoUtils.java index 04369730723d..1c8bcc9a037f 100644 --- a/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/utils/KryoUtils.java +++ b/gradoop-store/gradoop-accumulo/src/main/java/org/gradoop/storage/utils/KryoUtils.java @@ -18,10 +18,14 @@ import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; +import com.esotericsoftware.kryo.serializers.JavaSerializer; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; import java.util.Collection; import java.util.Map; @@ -41,6 +45,9 @@ public final class KryoUtils { kryo.setReferences(false); kryo.register(Collection.class); kryo.register(Map.class); + kryo.register(LocalDateTime.class, new JavaSerializer()); + kryo.register(LocalTime.class, new JavaSerializer()); + kryo.register(LocalDate.class, new JavaSerializer()); return kryo; }); } diff --git a/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/NOTICE b/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..309d7e764fa7 --- /dev/null +++ b/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/NOTICE @@ -0,0 +1,16 @@ +gradoop-accumulo +Copyright 2014-2019 Leipzig University (Database Research Group) + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.flink:flink-gelly_2.11:1.7.0 +- org.apache.flink:flink-hadoop-compatibility_2.11:1.7.0 +- org.apache.flink:flink-hbase_2.11:1.7.0 +- org.apache.flink:flink-java:1.7.0 +- org.apache.flink:flink-shaded-hadoop2:1.7.0 +- org.objenesis:objenesis:2.5.1 + +This project bundles the following dependencies under the BSD license. + +- com.esotericsoftware:kryo:4.0.2 +- com.esotericsoftware:minlog:1.3.0 diff --git a/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/licenses/LICENSE.kryo b/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/licenses/LICENSE.kryo new file mode 100644 index 000000000000..b041b0877512 --- /dev/null +++ b/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/licenses/LICENSE.kryo @@ -0,0 +1,10 @@ +Copyright (c) 2008-2018, Nathan Sweet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/licenses/LICENSE.minlog b/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/licenses/LICENSE.minlog new file mode 100644 index 000000000000..e1cd88478edf --- /dev/null +++ b/gradoop-store/gradoop-accumulo/src/main/resources/META-INF/licenses/LICENSE.minlog @@ -0,0 +1,10 @@ +Copyright (c) 2008, Nathan Sweet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/AccumuloStoreTestBase.java b/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/AccumuloStoreTestBase.java index 948954b1bde5..6094e07c845a 100644 --- a/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/AccumuloStoreTestBase.java +++ b/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/AccumuloStoreTestBase.java @@ -96,13 +96,23 @@ protected List sample( return ret; } + /** + * Wraps a test function. + */ public interface SocialTestContext { + /** + * Run the test. + * + * @param loader The loader used for access to a test graph. + * @param store The store instance to test. + * @param config The gradoop flink config used to run tests. + * @throws Throwable an Exception thrown by the test. + */ void test( AsciiGraphLoader loader, AccumuloEPGMStore store, GradoopFlinkConfig config ) throws Throwable; - } } diff --git a/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/AccumuloTestSuite.java b/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/AccumuloTestSuite.java index a862724800a3..580a2bb922e0 100644 --- a/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/AccumuloTestSuite.java +++ b/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/AccumuloTestSuite.java @@ -123,6 +123,8 @@ public static GradoopAccumuloConfig getAcConfig(String prefix) { /** * Create mini cluster accumulo instance for test + * + * @throws Exception on failure */ @BeforeClass public static void setupAccumulo() throws Exception { @@ -141,6 +143,8 @@ public static void setupAccumulo() throws Exception { /** * Terminate and remove temporary file + * + * @throws Exception on failure */ @AfterClass public static void terminateAccumulo() throws Exception { diff --git a/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/basic/StoreTest.java b/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/basic/StoreTest.java index b61e09e9fc7a..f741cc3ebec3 100644 --- a/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/basic/StoreTest.java +++ b/gradoop-store/gradoop-accumulo/src/test/java/org/gradoop/storage/impl/accumulo/basic/StoreTest.java @@ -19,6 +19,7 @@ import com.google.common.collect.Queues; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; import org.gradoop.common.GradoopTestUtils; import org.gradoop.common.config.GradoopConfig; import org.gradoop.common.exceptions.UnsupportedTypeException; @@ -83,8 +84,7 @@ import static org.gradoop.common.GradoopTestUtils.validateEPGMElements; import static org.gradoop.common.GradoopTestUtils.validateEPGMGraphElementCollections; import static org.gradoop.common.GradoopTestUtils.validateEPGMGraphElements; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * Accumulo graph store test @@ -98,6 +98,66 @@ public class StoreTest extends AccumuloStoreTestBase { private static final String TEST04 = "basic_04"; private static final String TEST05 = "basic_05"; + /** + * Creates tables, deletes them and checks if they were deleted. + * + * @throws AccumuloSecurityException when the user does not have permissions for some actions. + * @throws AccumuloException on general accumulo errors. + * @throws IOException when deleting tables fails. + */ + @Test + public void deleteTablesTest() throws AccumuloSecurityException, AccumuloException, IOException { + final String table = "test_to_delete"; + GradoopAccumuloConfig config = AccumuloTestSuite.getAcConfig(table); + AccumuloEPGMStore store = new AccumuloEPGMStore(config); + Connector connector = store.createConnector(); + // Make sure that the tables were created. + assertTrue(connector.tableOperations().exists(store.getVertexTableName())); + assertTrue(connector.tableOperations().exists(store.getEdgeTableName())); + assertTrue(connector.tableOperations().exists(store.getGraphHeadName())); + // Delete tables. + store.dropTables(); + // Check if they were deleted. + assertFalse(connector.tableOperations().exists(store.getVertexTableName())); + assertFalse(connector.tableOperations().exists(store.getEdgeTableName())); + assertFalse(connector.tableOperations().exists(store.getGraphHeadName())); + } + + /** + * Stores a graph, truncates tables and checks if the tables still exist and if they are empty. + * + * @throws AccumuloSecurityException when the user does not have permissions for some actions. + * @throws AccumuloException on general accumulo errors. + */ + @Test + public void truncateTablesTest() throws AccumuloSecurityException, AccumuloException, + IOException { + final String table = "test_to_truncate"; + GradoopAccumuloConfig config = AccumuloTestSuite.getAcConfig(table); + + AccumuloEPGMStore graphStore = new AccumuloEPGMStore(config); + + AsciiGraphLoader loader = getMinimalFullFeaturedGraphLoader(); + + GraphHead graphHead = loader.getGraphHeads().iterator().next(); + Vertex vertex = loader.getVertices().iterator().next(); + Edge edge = loader.getEdges().iterator().next(); + + graphStore.writeGraphHead(graphHead); + graphStore.writeVertex(vertex); + graphStore.writeEdge(edge); + + // re-open + graphStore.close(); + graphStore = new AccumuloEPGMStore(config); + graphStore.truncateTables(); + + assertFalse(graphStore.getVertexSpace().hasNext()); + assertFalse(graphStore.getEdgeSpace().hasNext()); + assertFalse(graphStore.getGraphSpace().hasNext()); + graphStore.close(); + } + /** * Creates persistent graph, vertex and edge data. Writes data to Accumulo, * closes the store, opens it and reads/validates the data again. diff --git a/gradoop-store/gradoop-hbase/pom.xml b/gradoop-store/gradoop-hbase/pom.xml index 31f40413568b..211754c34c64 100644 --- a/gradoop-store/gradoop-hbase/pom.xml +++ b/gradoop-store/gradoop-hbase/pom.xml @@ -5,7 +5,7 @@ gradoop-store org.gradoop - 0.4.4 + 0.4.5 gradoop-hbase diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/HBaseEPGMStore.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/HBaseEPGMStore.java index a669a79cf454..f9c88b1289bf 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/HBaseEPGMStore.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/HBaseEPGMStore.java @@ -40,11 +40,11 @@ import org.gradoop.storage.impl.hbase.api.EdgeHandler; import org.gradoop.storage.impl.hbase.api.GraphHeadHandler; import org.gradoop.storage.impl.hbase.api.VertexHandler; -import org.gradoop.storage.impl.hbase.predicate.filter.HBaseFilterUtils; -import org.gradoop.storage.impl.hbase.predicate.filter.api.HBaseElementFilter; import org.gradoop.storage.impl.hbase.iterator.HBaseEdgeIterator; import org.gradoop.storage.impl.hbase.iterator.HBaseGraphIterator; import org.gradoop.storage.impl.hbase.iterator.HBaseVertexIterator; +import org.gradoop.storage.impl.hbase.predicate.filter.HBaseFilterUtils; +import org.gradoop.storage.impl.hbase.predicate.filter.api.HBaseElementFilter; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -115,41 +115,26 @@ public HBaseEPGMStore( this.admin = Preconditions.checkNotNull(admin); } - /** - * {@inheritDoc} - */ @Override public GradoopHBaseConfig getConfig() { return config; } - /** - * {@inheritDoc} - */ @Override public String getVertexTableName() { return vertexTable.getName().getNameAsString(); } - /** - * {@inheritDoc} - */ @Override public String getEdgeTableName() { return edgeTable.getName().getNameAsString(); } - /** - * {@inheritDoc} - */ @Override public String getGraphHeadName() { return graphHeadTable.getName().getNameAsString(); } - /** - * {@inheritDoc} - */ @Override public void writeGraphHead(@Nonnull final EPGMGraphHead graphHead) throws IOException { GraphHeadHandler graphHeadHandler = config.getGraphHeadHandler(); @@ -164,9 +149,6 @@ public void writeGraphHead(@Nonnull final EPGMGraphHead graphHead) throws IOExce } } - /** - * {@inheritDoc} - */ @Override public void writeVertex(@Nonnull final EPGMVertex vertexData) throws IOException { VertexHandler vertexHandler = config.getVertexHandler(); @@ -181,9 +163,6 @@ public void writeVertex(@Nonnull final EPGMVertex vertexData) throws IOException } } - /** - * {@inheritDoc} - */ @Override public void writeEdge(@Nonnull final EPGMEdge edgeData) throws IOException { // write to table @@ -198,9 +177,6 @@ public void writeEdge(@Nonnull final EPGMEdge edgeData) throws IOException { } } - /** - * {@inheritDoc} - */ @Override public GraphHead readGraph(@Nonnull final GradoopId graphId) throws IOException { GraphHead graphData = null; @@ -224,9 +200,6 @@ public GraphHead readGraph(@Nonnull final GradoopId graphId) throws IOException return graphData; } - /** - * {@inheritDoc} - */ @Override public Vertex readVertex(@Nonnull final GradoopId vertexId) throws IOException { Vertex vertexData = null; @@ -277,9 +250,6 @@ public Edge readEdge(@Nonnull final GradoopId edgeId) throws IOException { return edgeData; } - /** - * {@inheritDoc} - */ @Nonnull @Override public ClosableIterator getGraphSpace( @@ -297,9 +267,6 @@ public ClosableIterator getGraphSpace( return new HBaseGraphIterator(graphHeadTable.getScanner(scan), config.getGraphHeadHandler()); } - /** - * {@inheritDoc} - */ @Nonnull @Override public ClosableIterator getVertexSpace( @@ -317,9 +284,6 @@ public ClosableIterator getVertexSpace( return new HBaseVertexIterator(vertexTable.getScanner(scan), config.getVertexHandler()); } - /** - * {@inheritDoc} - */ @Nonnull @Override public ClosableIterator getEdgeSpace( @@ -337,17 +301,11 @@ public ClosableIterator getEdgeSpace( return new HBaseEdgeIterator(edgeTable.getScanner(scan), config.getEdgeHandler()); } - /** - * {@inheritDoc} - */ @Override public void setAutoFlush(boolean autoFlush) { this.autoFlush = autoFlush; } - /** - * {@inheritDoc} - */ @Override public void flush() throws IOException { admin.flush(vertexTable.getName()); @@ -355,9 +313,6 @@ public void flush() throws IOException { admin.flush(graphHeadTable.getName()); } - /** - * {@inheritDoc} - */ @Override public void close() throws IOException { vertexTable.close(); @@ -380,6 +335,21 @@ public void dropTables() throws IOException { admin.deleteTable(graphHeadTable.getName()); } + /** + * First disable, then truncate all tables handled by this store instance, i.e. delete all rows. + * + * @throws IOException when truncating any table fails. + */ + public void truncateTables() throws IOException { + admin.disableTable(graphHeadTable.getName()); + admin.disableTable(vertexTable.getName()); + admin.disableTable(edgeTable.getName()); + + admin.truncateTable(getConfig().getGraphTableName(), true); + admin.truncateTable(getConfig().getVertexTableName(), true); + admin.truncateTable(getConfig().getEdgeTableName(), true); + } + /** * Attach a HBase filter represented by the given query to the given scan instance. * diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/api/ElementHandler.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/api/ElementHandler.java index 8e4f8aaaa5e4..773c74a55b6e 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/api/ElementHandler.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/api/ElementHandler.java @@ -40,6 +40,7 @@ public interface ElementHandler extends Serializable { * * @param gradoopId the gradoop id used to create row key from * @return persistent entity identifier + * @throws IOException on failure */ byte[] getRowKey(@Nonnull final GradoopId gradoopId) throws IOException; @@ -85,6 +86,7 @@ public interface ElementHandler extends Serializable { * * @param res row result * @return entity identifier + * @throws IOException on failure */ GradoopId readId(@Nonnull final Result res) throws IOException; diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/api/GraphElementHandler.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/api/GraphElementHandler.java index 43e490f5e52d..311f47f94a70 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/api/GraphElementHandler.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/api/GraphElementHandler.java @@ -33,12 +33,12 @@ public interface GraphElementHandler extends ElementHandler { * @param put {@link Put} to add graph identifiers to * @param graphElement graph element * @return put with graph identifiers + * @throws IOException on failure */ Put writeGraphIds( final Put put, final EPGMGraphElement graphElement - ) throws - IOException; + ) throws IOException; /** * Reads the graph identifiers from the given {@link Result}. diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/factory/HBaseEPGMStoreFactory.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/factory/HBaseEPGMStoreFactory.java index 31a8664a8109..df4fcf28dfef 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/factory/HBaseEPGMStoreFactory.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/factory/HBaseEPGMStoreFactory.java @@ -233,6 +233,7 @@ private static void deleteTablesIfExists( * * @param admin HBase admin * @param tableName name of the table to delete + * @throws IOException on failure */ private static void deleteTable(final Admin admin, final TableName tableName) throws IOException { admin.disableTable(tableName); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseEdgeHandler.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseEdgeHandler.java index a58697f3cc96..6739c0e6e0e6 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseEdgeHandler.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseEdgeHandler.java @@ -81,9 +81,6 @@ public HBaseEdgeHandler(EPGMEdgeFactory edgeFactory) { this.edgeFactory = edgeFactory; } - /** - * {@inheritDoc} - */ @Override public void createTable(final Admin admin, final HTableDescriptor tableDescriptor) throws IOException { @@ -101,41 +98,26 @@ public void createTable(final Admin admin, final HTableDescriptor tableDescripto } } - /** - * {@inheritDoc} - */ @Override public Put writeSource(final Put put, final GradoopId sourceId) { return put.addColumn(CF_META_BYTES, COL_SOURCE_BYTES, sourceId.toByteArray()); } - /** - * {@inheritDoc} - */ @Override public GradoopId readSourceId(Result res) { return GradoopId.fromByteArray(res.getValue(CF_META_BYTES, COL_SOURCE_BYTES)); } - /** - * {@inheritDoc} - */ @Override public Put writeTarget(Put put, GradoopId targetId) { return put.addColumn(CF_META_BYTES, COL_TARGET_BYTES, targetId.toByteArray()); } - /** - * {@inheritDoc} - */ @Override public GradoopId readTargetId(Result res) { return GradoopId.fromByteArray(res.getValue(CF_META_BYTES, COL_TARGET_BYTES)); } - /** - * {@inheritDoc} - */ @Override public Put writeEdge(Put put, EPGMEdge edgeData) { writeLabel(put, edgeData); @@ -146,27 +128,18 @@ public Put writeEdge(Put put, EPGMEdge edgeData) { return put; } - /** - * {@inheritDoc} - */ @Override public Edge readEdge(Result res) { return edgeFactory.initEdge(readId(res), readLabel(res), readSourceId(res), readTargetId(res), readProperties(res), readGraphIds(res)); } - /** - * {@inheritDoc} - */ @Override public EdgeHandler applyQuery(ElementQuery> query) { this.edgeQuery = query; return this; } - /** - * {@inheritDoc} - */ @Override public ElementQuery> getQuery() { return this.edgeQuery; diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseElementHandler.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseElementHandler.java index d3e1969746e4..9f44ff2ac4e3 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseElementHandler.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseElementHandler.java @@ -90,18 +90,12 @@ public GradoopId readId(@Nonnull final Result res) { } } - /** - * {@inheritDoc} - */ @Override public Put writeLabel(final Put put, final EPGMElement entity) { return (entity.getLabel() == null) ? put : put.addColumn(CF_META_BYTES, COL_LABEL_BYTES, Bytes.toBytes(entity.getLabel())); } - /** - * {@inheritDoc} - */ @Override public Put writeProperty(final Put put, Property property) { byte[] type = PropertyValueUtils.Bytes.getTypeByte(property.getValue()); @@ -111,9 +105,6 @@ public Put writeProperty(final Put put, Property property) { return put; } - /** - * {@inheritDoc} - */ @Override public Put writeProperties(final Put put, final EPGMElement entity) { if (entity.getProperties() != null && entity.getPropertyCount() > 0) { @@ -124,17 +115,11 @@ public Put writeProperties(final Put put, final EPGMElement entity) { return put; } - /** - * {@inheritDoc} - */ @Override public String readLabel(final Result res) { return Bytes.toString(res.getValue(CF_META_BYTES, COL_LABEL_BYTES)); } - /** - * {@inheritDoc} - */ @Override public Properties readProperties(final Result res) { Properties properties = Properties.create(); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseGraphHeadHandler.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseGraphHeadHandler.java index fbdbee4e26a8..27b4e6f959a1 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseGraphHeadHandler.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseGraphHeadHandler.java @@ -69,9 +69,6 @@ public HBaseGraphHeadHandler(EPGMGraphHeadFactory graphHeadFactory) { this.graphHeadFactory = graphHeadFactory; } - /** - * {@inheritDoc} - */ @Override public void createTable(final Admin admin, final HTableDescriptor tableDescriptor) throws IOException { @@ -81,9 +78,6 @@ public void createTable(final Admin admin, final HTableDescriptor tableDescripto admin.createTable(tableDescriptor); } - /** - * {@inheritDoc} - */ @Override public Put writeGraphHead(final Put put, final EPGMGraphHead graphData) { writeLabel(put, graphData); @@ -91,26 +85,17 @@ public Put writeGraphHead(final Put put, final EPGMGraphHead graphData) { return put; } - /** - * {@inheritDoc} - */ @Override public GraphHead readGraphHead(final Result res) { return graphHeadFactory.initGraphHead(readId(res), readLabel(res), readProperties(res)); } - /** - * {@inheritDoc} - */ @Override public GraphHeadHandler applyQuery(ElementQuery> query) { this.graphQuery = query; return this; } - /** - * {@inheritDoc} - */ @Override public ElementQuery> getQuery() { return this.graphQuery; diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseVertexHandler.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseVertexHandler.java index 4328f5fecc40..1f9ac779ef1b 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseVertexHandler.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/handler/HBaseVertexHandler.java @@ -70,9 +70,6 @@ public HBaseVertexHandler(EPGMVertexFactory vertexFactory) { this.vertexFactory = vertexFactory; } - /** - * {@inheritDoc} - */ @Override public void createTable(final Admin admin, final HTableDescriptor tableDescriptor) throws IOException { @@ -90,9 +87,6 @@ public void createTable(final Admin admin, final HTableDescriptor tableDescripto } } - /** - * {@inheritDoc} - */ @Override public Put writeVertex(Put put, EPGMVertex vertexData) { writeLabel(put, vertexData); @@ -101,27 +95,18 @@ public Put writeVertex(Put put, EPGMVertex vertexData) { return put; } - /** - * {@inheritDoc} - */ @Override public Vertex readVertex(final Result res) { return vertexFactory.initVertex(readId(res), readLabel(res), readProperties(res), readGraphIds(res)); } - /** - * {@inheritDoc} - */ @Override public VertexHandler applyQuery(ElementQuery> query) { this.vertexQuery = query; return this; } - /** - * {@inheritDoc} - */ @Override public ElementQuery> getQuery() { return this.vertexQuery; diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/HBaseDataSink.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/HBaseDataSink.java index b4ffc4950532..b2ea9e84bf84 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/HBaseDataSink.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/HBaseDataSink.java @@ -15,7 +15,6 @@ */ package org.gradoop.storage.impl.hbase.io; -import org.apache.commons.lang.NotImplementedException; import org.apache.flink.api.java.hadoop.mapreduce.HadoopOutputFormat; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.mapreduce.Job; @@ -50,37 +49,25 @@ public HBaseDataSink( super(epgmStore, flinkConfig); } - /** - * {@inheritDoc} - */ @Override public void write(LogicalGraph logicalGraph) throws IOException { write(logicalGraph, false); } - /** - * {@inheritDoc} - */ @Override public void write(GraphCollection graphCollection) throws IOException { write(graphCollection, false); } - /** - * {@inheritDoc} - */ @Override public void write(LogicalGraph logicalGraph, boolean overwrite) throws IOException { write(getFlinkConfig().getGraphCollectionFactory().fromGraph(logicalGraph), overwrite); } - /** - * {@inheritDoc} - */ @Override public void write(GraphCollection graphCollection, boolean overWrite) throws IOException { if (overWrite) { - throw new NotImplementedException("Overwriting graphs is not implemented in this sink."); + getStore().truncateTables(); } // transform graph data to persistent graph data and write it diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/HBaseDataSource.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/HBaseDataSource.java index b47f0e6930ef..c027bd7ff6e6 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/HBaseDataSource.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/HBaseDataSource.java @@ -98,17 +98,11 @@ private HBaseDataSource( this.edgeQuery = edgeQuery; } - /** - * {@inheritDoc} - */ @Override public LogicalGraph getLogicalGraph() { return getGraphCollection().reduce(new ReduceCombination()); } - /** - * {@inheritDoc} - */ @Override public GraphCollection getGraphCollection() { GradoopFlinkConfig config = getFlinkConfig(); @@ -138,9 +132,6 @@ public GraphCollection getGraphCollection() { return config.getGraphCollectionFactory().fromDataSets(graphHeads, vertices, edges); } - /** - * {@inheritDoc} - */ @Nonnull @Override public HBaseDataSource applyGraphPredicate( @@ -149,9 +140,6 @@ public HBaseDataSource applyGraphPredicate( return new HBaseDataSource(getStore(), getFlinkConfig(), query, vertexQuery, edgeQuery); } - /** - * {@inheritDoc} - */ @Nonnull @Override public HBaseDataSource applyVertexPredicate( @@ -160,9 +148,6 @@ public HBaseDataSource applyVertexPredicate( return new HBaseDataSource(getStore(), getFlinkConfig(), graphHeadQuery, query, edgeQuery); } - /** - * {@inheritDoc} - */ @Nonnull @Override public HBaseDataSource applyEdgePredicate( @@ -171,9 +156,6 @@ public HBaseDataSource applyEdgePredicate( return new HBaseDataSource(getStore(), getFlinkConfig(), graphHeadQuery, vertexQuery, query); } - /** - * {@inheritDoc} - */ @Override public boolean isFilterPushedDown() { return this.graphHeadQuery != null || this.vertexQuery != null || this.edgeQuery != null; diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildEdgeMutation.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildEdgeMutation.java index 22a0deafaead..15fb0861b324 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildEdgeMutation.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildEdgeMutation.java @@ -54,9 +54,6 @@ public BuildEdgeMutation(EdgeHandler edgeHandler) { this.edgeHandler = edgeHandler; } - /** - * {@inheritDoc} - */ @Override public Tuple2 map(Edge edge) throws Exception { GradoopId key = edge.getId(); @@ -68,9 +65,6 @@ public Tuple2 map(Edge edge) throws Exception { return reuseTuple; } - /** - * {@inheritDoc} - */ @Override public void open(Configuration parameters) throws Exception { super.open(parameters); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildGraphHeadMutation.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildGraphHeadMutation.java index 0ae1ff4f228b..ceac5ae97042 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildGraphHeadMutation.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildGraphHeadMutation.java @@ -55,18 +55,12 @@ public BuildGraphHeadMutation(GraphHeadHandler graphHeadHandler) { this.graphHeadHandler = graphHeadHandler; } - /** - * {@inheritDoc} - */ @Override public void open(Configuration parameters) throws Exception { super.open(parameters); reuseTuple = new Tuple2<>(); } - /** - * {@inheritDoc} - */ @Override public Tuple2 map(GraphHead graphHead) throws Exception { GradoopId key = graphHead.getId(); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildVertexMutation.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildVertexMutation.java index 036a086a36f9..cd31ed2b954b 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildVertexMutation.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/functions/BuildVertexMutation.java @@ -53,18 +53,12 @@ public BuildVertexMutation(VertexHandler vertexHandler) { this.vertexHandler = vertexHandler; } - /** - * {@inheritDoc} - */ @Override public void open(Configuration parameters) throws Exception { super.open(parameters); reuseTuple = new Tuple2<>(); } - /** - * {@inheritDoc} - */ @Override public Tuple2 map(Vertex vertex) throws Exception { GradoopId key = vertex.getId(); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/EdgeTableInputFormat.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/EdgeTableInputFormat.java index f4aac5f4e733..bac6b26d65f9 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/EdgeTableInputFormat.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/EdgeTableInputFormat.java @@ -66,17 +66,11 @@ protected Scan getScanner() { return scan; } - /** - * {@inheritDoc} - */ @Override protected String getTableName() { return edgeTableName; } - /** - * {@inheritDoc} - */ @Override protected Tuple1 mapResultToTuple(Result result) { return new Tuple1<>(edgeHandler.readEdge(result)); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/GraphHeadTableInputFormat.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/GraphHeadTableInputFormat.java index 7a624a76e36e..0e13faa83b8e 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/GraphHeadTableInputFormat.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/GraphHeadTableInputFormat.java @@ -67,17 +67,11 @@ protected Scan getScanner() { return scan; } - /** - * {@inheritDoc} - */ @Override protected String getTableName() { return graphHeadTableName; } - /** - * {@inheritDoc} - */ @Override protected Tuple1 mapResultToTuple(Result result) { return new Tuple1<>(graphHeadHandler.readGraphHead(result)); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/VertexTableInputFormat.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/VertexTableInputFormat.java index 5e1f6485ffe3..701f2a2f8888 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/VertexTableInputFormat.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/io/inputformats/VertexTableInputFormat.java @@ -66,17 +66,11 @@ protected Scan getScanner() { return scan; } - /** - * {@inheritDoc} - */ @Override protected String getTableName() { return vertexTableName; } - /** - * {@inheritDoc} - */ @Override protected Tuple1 mapResultToTuple(Result result) { return new Tuple1<>(vertexHandler.readVertex(result)); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/api/HBaseElementFilter.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/api/HBaseElementFilter.java index f15776d017ce..bceca0b60b1e 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/api/HBaseElementFilter.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/api/HBaseElementFilter.java @@ -33,27 +33,18 @@ public interface HBaseElementFilter extends ElementFilter>, Serializable { - /** - * {@inheritDoc} - */ @Nonnull @Override default HBaseElementFilter or(@Nonnull HBaseElementFilter another) { return Or.create(this, another); } - /** - * {@inheritDoc} - */ @Nonnull @Override default HBaseElementFilter and(@Nonnull HBaseElementFilter another) { return And.create(this, another); } - /** - * {@inheritDoc} - */ @Nonnull @Override default HBaseElementFilter negate() { diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/And.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/And.java index 7ac1c6284b42..92f405393123 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/And.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/And.java @@ -65,9 +65,6 @@ public static And create(HBaseElementFilter... pre return new And<>(formula); } - /** - * {@inheritDoc} - */ @Nonnull @Override public Filter toHBaseFilter(boolean negate) { @@ -83,9 +80,6 @@ public Filter toHBaseFilter(boolean negate) { return filterList; } - /** - * {@inheritDoc} - */ @Override public String toString() { StringJoiner joiner = new StringJoiner(" AND "); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/Not.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/Not.java index de80167243de..8969407a189d 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/Not.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/Not.java @@ -53,9 +53,6 @@ public static Not of(HBaseElementFilter predicate) return new Not<>(predicate); } - /** - * {@inheritDoc} - */ @Nonnull @Override public Filter toHBaseFilter(boolean negate) { @@ -63,9 +60,6 @@ public Filter toHBaseFilter(boolean negate) { return predicate.toHBaseFilter(!negate); } - /** - * {@inheritDoc} - */ @Override public String toString() { return "NOT " + predicate; diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/Or.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/Or.java index 7d3127f45573..ed183659e1f9 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/Or.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/calculate/Or.java @@ -65,9 +65,6 @@ public static Or create(HBaseElementFilter... pred return new Or<>(formula); } - /** - * {@inheritDoc} - */ @Nonnull @Override public Filter toHBaseFilter(boolean negate) { @@ -83,9 +80,6 @@ public Filter toHBaseFilter(boolean negate) { return filterList; } - /** - * {@inheritDoc} - */ @Override public String toString() { StringJoiner joiner = new StringJoiner(" OR "); diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBaseLabelIn.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBaseLabelIn.java index 84eb9c715111..ee44ab52ec1a 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBaseLabelIn.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBaseLabelIn.java @@ -40,9 +40,6 @@ public HBaseLabelIn(String... labels) { super(labels); } - /** - * {@inheritDoc} - */ @Nonnull @Override public Filter toHBaseFilter(boolean negate) { diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBaseLabelReg.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBaseLabelReg.java index c88b938ae3eb..c7d978d63d4b 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBaseLabelReg.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBaseLabelReg.java @@ -41,9 +41,6 @@ public HBaseLabelReg(Pattern reg) { super(reg); } - /** - * {@inheritDoc} - */ @Nonnull @Override public Filter toHBaseFilter(boolean negate) { diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropEquals.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropEquals.java index 5b1140844410..920e4a1e0f39 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropEquals.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropEquals.java @@ -41,9 +41,6 @@ public HBasePropEquals(@Nonnull String key, @Nonnull Object value) { super(key, value); } - /** - * {@inheritDoc} - */ @Override @Nonnull public Filter toHBaseFilter(boolean negate) { diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropLargerThan.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropLargerThan.java index 2ee5f5634941..0e9239598d17 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropLargerThan.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropLargerThan.java @@ -42,9 +42,6 @@ public HBasePropLargerThan(@Nonnull String key, @Nonnull Object min, boolean inc super(key, min, include); } - /** - * {@inheritDoc} - */ @Nonnull @Override public Filter toHBaseFilter(boolean negate) { diff --git a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropReg.java b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropReg.java index 13f524078144..48f79d010117 100644 --- a/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropReg.java +++ b/gradoop-store/gradoop-hbase/src/main/java/org/gradoop/storage/impl/hbase/predicate/filter/impl/HBasePropReg.java @@ -42,9 +42,6 @@ public HBasePropReg(@Nonnull String key, @Nonnull Pattern reg) { super(key, reg); } - /** - * {@inheritDoc} - */ @Nonnull @Override public Filter toHBaseFilter(boolean negate) { diff --git a/gradoop-store/gradoop-hbase/src/main/resources/META-INF/NOTICE b/gradoop-store/gradoop-hbase/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..835dbc3dda97 --- /dev/null +++ b/gradoop-store/gradoop-hbase/src/main/resources/META-INF/NOTICE @@ -0,0 +1,9 @@ +gradoop-hbase +Copyright 2014-2019 Leipzig University (Database Research Group) + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.github.s1ck:gdl:0.3 +- org.codehaus.jettison:jettison:1.3.7 +- org.apache.hbase:hbase-hadoop-compat:1.4.3 +- org.apache.hbase:hbase-server:1.4.3 diff --git a/gradoop-store/gradoop-hbase/src/main/resources/META-INF/licenses/LICENSE.HBaseWD b/gradoop-store/gradoop-hbase/src/main/resources/META-INF/licenses/LICENSE.HBaseWD new file mode 100644 index 000000000000..ac907a8a9e54 --- /dev/null +++ b/gradoop-store/gradoop-hbase/src/main/resources/META-INF/licenses/LICENSE.HBaseWD @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 William Kapk + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseDefaultGraphStoreTest.java b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseDefaultGraphStoreTest.java index f85c1e839232..f826782bf078 100644 --- a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseDefaultGraphStoreTest.java +++ b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseDefaultGraphStoreTest.java @@ -56,6 +56,8 @@ public class HBaseDefaultGraphStoreTest extends GradoopHBaseTestBase { /** * Instantiate the EPGMStore with a prefix and persist social media data + * + * @throws IOException on failure */ @BeforeClass public static void setUp() throws IOException { @@ -65,6 +67,8 @@ public static void setUp() throws IOException { /** * Closes the static EPGMStore + * + * @throws IOException on failure */ @AfterClass public static void tearDown() throws IOException { @@ -76,6 +80,8 @@ public static void tearDown() throws IOException { /** * Test the getGraphSpace() method with an id filter predicate + * + * @throws IOException on failure */ @Test public void testGetGraphSpaceWithIdPredicate() throws IOException { @@ -99,6 +105,8 @@ public void testGetGraphSpaceWithIdPredicate() throws IOException { /** * Test the getGraphSpace() method without an id filter predicate + * + * @throws IOException on failure */ @Test public void testGetGraphSpaceWithoutIdPredicate() throws IOException { @@ -116,6 +124,8 @@ public void testGetGraphSpaceWithoutIdPredicate() throws IOException { /** * Test the getVertexSpace() method with an id filter predicate + * + * @throws IOException on failure */ @Test public void testGetVertexSpaceWithIdPredicate() throws IOException { @@ -140,6 +150,8 @@ public void testGetVertexSpaceWithIdPredicate() throws IOException { /** * Test the getVertexSpace() method without an id filter predicate + * + * @throws IOException on failure */ @Test public void testGetVertexSpaceWithoutIdPredicate() throws IOException { @@ -157,6 +169,8 @@ public void testGetVertexSpaceWithoutIdPredicate() throws IOException { /** * Test the getEdgeSpace() method with an id filter predicate + * + * @throws IOException on failure */ @Test public void testGetEdgeSpaceWithIdPredicate() throws IOException { @@ -180,6 +194,8 @@ public void testGetEdgeSpaceWithIdPredicate() throws IOException { /** * Test the getEdgeSpace() method without an id filter predicate + * + * @throws IOException on failure */ @Test public void testGetEdgeSpaceWithoutIdPredicate() throws IOException { @@ -198,6 +214,8 @@ public void testGetEdgeSpaceWithoutIdPredicate() throws IOException { /** * Test the getGraphSpace(), getVertexSpace() and getEdgeSpace() method * with the {@link HBaseLabelIn} predicate + * + * @throws IOException on failure */ @Test public void testGetElementSpaceWithLabelInPredicate() throws IOException { @@ -245,6 +263,8 @@ public void testGetElementSpaceWithLabelInPredicate() throws IOException { /** * Test the getGraphSpace(), getVertexSpace() and getEdgeSpace() method * with the {@link HBaseLabelReg} predicate + * + * @throws IOException on failure */ @Test public void testGetElementSpaceWithLabelRegPredicate() throws IOException { @@ -291,6 +311,8 @@ public void testGetElementSpaceWithLabelRegPredicate() throws IOException { /** * Test the getGraphSpace(), getVertexSpace() and getEdgeSpace() method * with the {@link HBasePropEquals} predicate + * + * @throws IOException on failure */ @Test public void testGetElementSpaceWithPropEqualsPredicate() throws IOException { @@ -345,6 +367,8 @@ public void testGetElementSpaceWithPropEqualsPredicate() throws IOException { /** * Test the getGraphSpace(), getVertexSpace() and getEdgeSpace() method * with the {@link HBasePropLargerThan} predicate + * + * @throws IOException on failure */ @Test public void testGetElementSpaceWithPropLargerThanPredicate() throws IOException { @@ -403,6 +427,8 @@ public void testGetElementSpaceWithPropLargerThanPredicate() throws IOException /** * Test the getGraphSpace(), getVertexSpace() and getEdgeSpace() method * with the {@link HBasePropReg} predicate + * + * @throws IOException on failure */ @Test public void testGetElementSpaceWithPropRegPredicate() throws IOException { @@ -460,6 +486,8 @@ public void testGetElementSpaceWithPropRegPredicate() throws IOException { /** * Test the getGraphSpace(), getVertexSpace() and getEdgeSpace() method * with complex predicates + * + * @throws IOException on failure */ @Test public void testGetElementSpaceWithChainedPredicates() throws IOException { diff --git a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseEPGMStoreTest.java b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseEPGMStoreTest.java index e972f51baad0..7d790252ad23 100644 --- a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseEPGMStoreTest.java +++ b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseEPGMStoreTest.java @@ -31,14 +31,17 @@ import org.gradoop.common.model.impl.pojo.VertexFactory; import org.gradoop.common.model.impl.properties.Properties; import org.gradoop.common.util.AsciiGraphLoader; +import org.gradoop.storage.common.iterator.ClosableIterator; import org.junit.Test; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Queue; import static org.gradoop.common.GradoopTestUtils.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -49,6 +52,8 @@ public class HBaseEPGMStoreTest extends GradoopHBaseTestBase { /** * Creates persistent graph, vertex and edge data. Writes data to HBase, * closes the store, opens it and reads/validates the data again. + * + * @throws IOException on failure */ @Test public void writeCloseOpenReadTest() throws IOException { @@ -78,6 +83,8 @@ public void writeCloseOpenReadTest() throws IOException { /** * Creates persistent graph, vertex and edge data. Writes data to HBase, * closes the store, opens it and reads/validates the data again. + * + * @throws IOException on failure */ @Test public void writeCloseOpenReadTestWithPrefix() throws IOException { @@ -108,6 +115,8 @@ public void writeCloseOpenReadTestWithPrefix() throws IOException { /** * Creates persistent graph, vertex and edge data. Writes data to HBase, * flushes the tables and reads/validates the data. + * + * @throws IOException on failure */ @Test public void writeFlushReadTest() throws IOException { @@ -191,6 +200,8 @@ public void iteratorTest() throws IOException { /** * Tries to add an unsupported property type {@link Queue} as property value. + * + * @throws IOException on failure */ @Test(expected = UnsupportedTypeException.class) public void wrongPropertyTypeTest() throws IOException { @@ -215,6 +226,8 @@ public void wrongPropertyTypeTest() throws IOException { /** * Checks if property values are read correctly. + * + * @throws IOException on failure */ @SuppressWarnings("Duplicates") @Test @@ -313,6 +326,52 @@ public void propertyTypeTest() throws IOException { graphStore.close(); } + /** + * Test the truncate tables functionality. + */ + @Test + public void truncateTablesTest() throws IOException { + HBaseEPGMStore store = createEmptyEPGMStore("truncateTest"); + AsciiGraphLoader loader = getMinimalFullFeaturedGraphLoader(); + checkIfStoreIsEmpty("Store was not empty before writing anything.", store); + // Now write something to the store, check if it was written (i.e. the store is not empty). + GraphHead graphHead = loader.getGraphHeads().iterator().next(); + Vertex vertex = loader.getVertices().iterator().next(); + Edge edge = loader.getEdges().iterator().next(); + store.writeGraphHead(graphHead); + store.writeVertex(vertex); + store.writeEdge(edge); + store.flush(); + validateGraphHead(store, graphHead); + validateVertex(store, vertex); + validateEdge(store, edge); + // Now truncate and check if the store if empty afterwards. + store.truncateTables(); + checkIfStoreIsEmpty("Store was not empty after truncating.", store); + // Finally check if we can write elements. + store.writeGraphHead(graphHead); + store.writeVertex(vertex); + store.writeEdge(edge); + store.close(); + } + + /** + * Check if all tables of are store are empty. + * + * @param message The messenge used in the assertion. + * @param store The HBase store to check. + * @throws IOException when accessing the store fails. + */ + private void checkIfStoreIsEmpty(String message, HBaseEPGMStore store) throws IOException { + for (ClosableIterator space : Arrays.asList(store.getGraphSpace(), + store.getVertexSpace(), store.getEdgeSpace())) { + boolean hasNext = space.hasNext(); + // Make sure to close the iterator before the assertion. + space.close(); + assertFalse(message, hasNext); + } + } + private AsciiGraphLoader getMinimalFullFeaturedGraphLoader() { String asciiGraph = ":G{k:\"v\"}[(v:V{k:\"v\"}),(v)-[:e{k:\"v\"}]->(v)]"; return AsciiGraphLoader.fromString(asciiGraph, GradoopConfig.getDefaultConfig()); diff --git a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseSplitRegionGraphStoreTest.java b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseSplitRegionGraphStoreTest.java index 8374ceb380e7..b6bf8adde172 100644 --- a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseSplitRegionGraphStoreTest.java +++ b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseSplitRegionGraphStoreTest.java @@ -31,6 +31,8 @@ public class HBaseSplitRegionGraphStoreTest extends HBaseDefaultGraphStoreTest { /** * Instantiate the EPGMStore with a prefix and persist social media data. * In addition, use pre-split regions. + * + * @throws IOException on failure */ @BeforeClass public static void setUp() throws IOException { diff --git a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseSpreadingByteGraphStoreTest.java b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseSpreadingByteGraphStoreTest.java index 1afe810b2a03..64afe4ef53e2 100644 --- a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseSpreadingByteGraphStoreTest.java +++ b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/HBaseSpreadingByteGraphStoreTest.java @@ -32,6 +32,8 @@ public class HBaseSpreadingByteGraphStoreTest extends HBaseDefaultGraphStoreTest /** * Instantiate the EPGMStore with a prefix and persist social media data. * In addition, use a spreading byte as prefix of each row key. + * + * @throws IOException on failure */ @BeforeClass public static void setUp() throws IOException { diff --git a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/io/HBaseDataSinkSourceTest.java b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/io/HBaseDataSinkSourceTest.java index 7e12332960fb..c6b3d54c32d8 100644 --- a/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/io/HBaseDataSinkSourceTest.java +++ b/gradoop-store/gradoop-hbase/src/test/java/org/gradoop/storage/impl/hbase/io/HBaseDataSinkSourceTest.java @@ -16,7 +16,6 @@ package org.gradoop.storage.impl.hbase.io; import com.google.common.collect.Lists; -import org.apache.commons.lang.NotImplementedException; import org.apache.flink.api.java.io.LocalCollectionOutputFormat; import org.gradoop.common.GradoopTestUtils; import org.gradoop.common.model.api.entities.EPGMIdentifiable; @@ -27,6 +26,7 @@ import org.gradoop.common.model.impl.properties.PropertyValue; import org.gradoop.flink.model.GradoopFlinkTestBase; import org.gradoop.flink.model.impl.epgm.GraphCollection; +import org.gradoop.flink.model.impl.epgm.LogicalGraph; import org.gradoop.flink.util.FlinkAsciiGraphLoader; import org.gradoop.storage.common.predicate.query.Query; import org.gradoop.storage.config.GradoopHBaseConfig; @@ -75,6 +75,8 @@ public class HBaseDataSinkSourceTest extends GradoopFlinkTestBase { /** * Instantiate the EPGMStore with a prefix and persist social media data + * + * @throws IOException on failure */ @BeforeClass public static void setUp() throws IOException { @@ -96,6 +98,8 @@ public static void setUp() throws IOException { /** * Closes the static EPGMStore + * + * @throws IOException on failure */ @AfterClass public static void tearDown() throws IOException { @@ -169,6 +173,8 @@ public void testConfig() { /** * Test reading a graph collection from {@link HBaseDataSource} + * + * @throws IOException on failure */ @Test public void testReadFromSource() throws Exception { @@ -195,6 +201,8 @@ public void testReadFromSource() throws Exception { /** * Test reading a graph collection from {@link HBaseDataSource} with empty predicates + * + * @throws Exception on failure */ @Test public void testReadFromSourceWithEmptyPredicates() throws Exception { @@ -339,6 +347,8 @@ public void testReadWithEdgeIdPredicate() throws Throwable { /** * Test reading a graph collection from {@link HBaseDataSource} * with a {@link HBaseLabelIn} predicate on each graph element + * + * @throws Exception on failure */ @Test public void testReadWithLabelInPredicate() throws Exception { @@ -395,6 +405,8 @@ public void testReadWithLabelInPredicate() throws Exception { /** * Test reading a graph collection from {@link HBaseDataSource} * with a {@link HBaseLabelReg} predicate on each graph element + * + * @throws Exception on failure */ @Test public void testReadWithLabelRegPredicate() throws Exception { @@ -447,6 +459,8 @@ public void testReadWithLabelRegPredicate() throws Exception { /** * Test reading a graph collection from {@link HBaseDataSource} * with a {@link HBasePropEquals} predicate on each graph element + * + * @throws Exception on failure */ @Test public void testReadWithPropEqualsPredicate() throws Exception { @@ -507,6 +521,8 @@ public void testReadWithPropEqualsPredicate() throws Exception { /** * Test reading a graph collection from {@link HBaseDataSource} * with a {@link HBasePropLargerThan} predicate on each graph element + * + * @throws Exception on failure */ @Test public void testReadWithPropLargerThanPredicate() throws Exception { @@ -573,6 +589,8 @@ public void testReadWithPropLargerThanPredicate() throws Exception { /** * Test reading a graph collection from {@link HBaseDataSource} * with a {@link HBasePropReg} predicate on each graph element + * + * @throws Exception on failure */ @Test public void testReadWithPropRegPredicate() throws Exception { @@ -639,6 +657,8 @@ public void testReadWithPropRegPredicate() throws Exception { /** * Test reading a graph collection from {@link HBaseDataSource} * with logical chained predicates on each graph element + * + * @throws Exception on failure */ @Test public void testReadWithChainedPredicates() throws Exception { @@ -711,6 +731,8 @@ public void testReadWithChainedPredicates() throws Exception { /** * Test writing a graph to {@link HBaseDataSink} + * + * @throws Exception on failure */ @Test public void testWriteToSink() throws Exception { @@ -779,17 +801,24 @@ public void testWriteToSink() throws Exception { /** * Test writing a graph to {@link HBaseDataSink} with overwrite flag, that results in an exception + * + * @throws Exception on failure */ - @Test(expected = NotImplementedException.class) + @Test public void testWriteToSinkWithOverWrite() throws Exception { // Create an empty store - HBaseEPGMStore newStore = createEmptyEPGMStore("testWriteToSink"); - - GraphCollection graphCollection = getConfig().getGraphCollectionFactory() - .createEmptyCollection(); - - new HBaseDataSink(newStore, getConfig()).write(graphCollection, true); - + HBaseEPGMStore store = createEmptyEPGMStore("testWriteToSinkWithOverwrite"); + // Get a test graph and write it to the store. + FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); + LogicalGraph testGraph = loader.getLogicalGraphByVariable("g0"); + testGraph.writeTo(new HBaseDataSink(store, getConfig()), false); + getExecutionEnvironment().execute(); + // Now write a different graph with overwrite and validate it by reading it again. + LogicalGraph testGraph2 = loader.getLogicalGraphByVariable("g1"); + testGraph2.writeTo(new HBaseDataSink(store, getConfig()), true); getExecutionEnvironment().execute(); + collectAndAssertTrue(new HBaseDataSource(store, getConfig()).getLogicalGraph() + .equalsByElementData(testGraph2)); + store.close(); } } diff --git a/gradoop-store/gradoop-store-api/pom.xml b/gradoop-store/gradoop-store-api/pom.xml index 1760d5ff4a20..546db88ccc91 100644 --- a/gradoop-store/gradoop-store-api/pom.xml +++ b/gradoop-store/gradoop-store-api/pom.xml @@ -5,7 +5,7 @@ gradoop-store org.gradoop - 0.4.4 + 0.4.5 4.0.0 diff --git a/gradoop-store/gradoop-store-api/src/main/java/org/gradoop/storage/common/api/EPGMGraphPredictableOutput.java b/gradoop-store/gradoop-store-api/src/main/java/org/gradoop/storage/common/api/EPGMGraphPredictableOutput.java index f95cd00833a9..61719b0b65a9 100644 --- a/gradoop-store/gradoop-store-api/src/main/java/org/gradoop/storage/common/api/EPGMGraphPredictableOutput.java +++ b/gradoop-store/gradoop-store-api/src/main/java/org/gradoop/storage/common/api/EPGMGraphPredictableOutput.java @@ -64,6 +64,7 @@ default ClosableIterator getEdgeSpace(int cacheSize) throws IOException { * * @param query element query predicate * @return Graph Heads + * @throws IOException on failure */ @Nonnull default ClosableIterator getGraphSpace( @@ -78,6 +79,7 @@ default ClosableIterator getGraphSpace( * @param query element query predicate * @param cacheSize client result cache size * @return Graph Heads + * @throws IOException on failure */ @Nonnull ClosableIterator getGraphSpace( @@ -90,6 +92,7 @@ ClosableIterator getGraphSpace( * * @param query element query predicate * @return Vertices + * @throws IOException on failure */ @Nonnull default ClosableIterator getVertexSpace( @@ -104,6 +107,7 @@ default ClosableIterator getVertexSpace( * @param query element query predicate * @param cacheSize result cache size * @return Vertices + * @throws IOException on failure */ @Nonnull ClosableIterator getVertexSpace( @@ -116,6 +120,7 @@ ClosableIterator getVertexSpace( * * @param query element query predicate * @return edges + * @throws IOException on failure */ @Nonnull default ClosableIterator getEdgeSpace( @@ -130,6 +135,7 @@ default ClosableIterator getEdgeSpace( * @param query element query predicate * @param cacheSize result cache size * @return edges + * @throws IOException on failure */ @Nonnull ClosableIterator getEdgeSpace( diff --git a/gradoop-store/gradoop-store-api/src/main/resources/META-INF/NOTICE b/gradoop-store/gradoop-store-api/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..c949565b60e0 --- /dev/null +++ b/gradoop-store/gradoop-store-api/src/main/resources/META-INF/NOTICE @@ -0,0 +1,6 @@ +gradoop-store-api +Copyright 2014-2019 Leipzig University (Database Research Group) + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.flink:flink-java:1.7.0 diff --git a/gradoop-store/pom.xml b/gradoop-store/pom.xml index 15845f7ebe89..f0d0d80dc883 100644 --- a/gradoop-store/pom.xml +++ b/gradoop-store/pom.xml @@ -5,7 +5,7 @@ gradoop-parent org.gradoop - 0.4.4 + 0.4.5 4.0.0 diff --git a/licenses-binary/LICENSE.antlr4 b/licenses-binary/LICENSE.antlr4 new file mode 100644 index 000000000000..e00ceeefb8c6 --- /dev/null +++ b/licenses-binary/LICENSE.antlr4 @@ -0,0 +1,10 @@ +Copyright (c) 2012 Terence Parr and Sam Harwell +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.kryo b/licenses-binary/LICENSE.kryo new file mode 100644 index 000000000000..b041b0877512 --- /dev/null +++ b/licenses-binary/LICENSE.kryo @@ -0,0 +1,10 @@ +Copyright (c) 2008-2018, Nathan Sweet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.minlog b/licenses-binary/LICENSE.minlog new file mode 100644 index 000000000000..e1cd88478edf --- /dev/null +++ b/licenses-binary/LICENSE.minlog @@ -0,0 +1,10 @@ +Copyright (c) 2008, Nathan Sweet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/LICENSE.HBaseWD b/licenses/LICENSE.HBaseWD new file mode 100644 index 000000000000..ac907a8a9e54 --- /dev/null +++ b/licenses/LICENSE.HBaseWD @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 William Kapk + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/licenses/LICENSE.MongoDB b/licenses/LICENSE.MongoDB new file mode 100644 index 000000000000..2f44352e7353 --- /dev/null +++ b/licenses/LICENSE.MongoDB @@ -0,0 +1,87 @@ +1) Copyright 2008-2015 MongoDB, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +2) The following files: Immutable.java, NotThreadSafe.java, ThreadSafe.java + + Copyright (c) 2005 Brian Goetz and Tim Peierls + Released under the Creative Commons Attribution License (http://creativecommons.org/licenses/by/2.5) + Official home: http://www.jcip.net + + Any republication or derived work distributed in source code form + must include this copyright and license notice. + +3) The following files: Assertions.java, AbstractCopyOnWriteMap.java, CopyOnWriteMap.java + + Copyright (c) 2008-2014 Atlassian Pty Ltd + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +4) The following files: Beta.java + + Copyright 2010 The Guava Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +5) The following files: Base64Codec.java + + Copyright 1999,2005 The Apache Software Foundation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +6) The following files: ReadTimeoutHandler.java + + Copyright 2015 MongoDB, Inc. + Copyright 2012 The Netty Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pom.xml b/pom.xml index 7a07b2c98097..a22cc1b3a742 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.gradoop gradoop-parent pom - 0.4.4 + 0.4.5 Gradoop Parent http://www.gradoop.org @@ -106,6 +106,14 @@ + + + dbleipzig + DB Leipzig Archiva + https://wdiserv1.informatik.uni-leipzig.de:443/archiva/repository/dbleipzig/ + + + release_artifacts_gradoop @@ -129,23 +137,24 @@ 1.9.0 1.4 - 1.7.0 + 1.7.2 0.3 1.4.3 0.1.10 - 4.11 + 4.12 1.3.7 1.2.3 4.0.2 1.2.17 2.21.0 + 0.2.4-SNAPSHOT log4j-test.properties -Xmx1G -Dlog4j.configuration=${log4j.properties} 3.5.1 3.0.0 - 3.0.1 + 3.0.5 2.3.2 2.5.3 2.19.1 @@ -500,7 +509,9 @@ report - ${session.executionRootDirectory}/target/coverage-reports/${project.name} + + ${session.executionRootDirectory}/target/coverage-reports/${project.name} + ${project.name}

Code Coverage Report for Gradoop ${project.version}
@@ -615,10 +626,29 @@ com.esotericsoftware - kryo-shaded + kryo ${dep.kryo.version} + + + org.opencypher + flink-cypher + ${dep.okapi.version} + + + org.slf4j + slf4j-nop + + + + + + org.opencypher + spark-cypher + ${dep.okapi.version} + + com.github.s1ck