|
22 | 22 | <modelVersion>4.0.0</modelVersion> |
23 | 23 |
|
24 | 24 | <groupId>org.deeplearning4j</groupId> |
25 | | - <artifactId>data-pipeline-examples</artifactId> |
| 25 | + <artifactId>dl4j-distributed-training-examples</artifactId> |
26 | 26 | <version>1.0.0-beta7</version> |
27 | | - <name>Building a data pipeline prior to modeling</name> |
28 | | - <description>Loading raw data and processing it before training</description> |
| 27 | + <name>Introduction to Distributed Training with DL4J</name> |
| 28 | + <description>A set of examples introducing distributed training with the DL4J framework</description> |
29 | 29 |
|
30 | 30 | <properties> |
31 | | - <dl4j-master.version>1.0.0-SNAPSHOT</dl4j-master.version> |
| 31 | + <dl4j-master.version>1.0.0-beta7</dl4j-master.version> |
32 | 32 | <!-- Change the nd4j.backend property to nd4j-cuda-X-platform to use CUDA GPUs --> |
33 | | - <!-- <nd4j.backend>nd4j-cuda-10.2-platform</nd4j.backend> --> |
| 33 | + <!-- <nd4j.backend>nd4j-cuda-10.2-platform</nd4j.backend> --> |
34 | 34 | <nd4j.backend>nd4j-native</nd4j.backend> |
35 | 35 | <java.version>1.8</java.version> |
| 36 | + <shadedClassifier>bin</shadedClassifier> |
| 37 | + <scala.binary.version>2.11</scala.binary.version> |
36 | 38 | <maven-compiler-plugin.version>3.6.1</maven-compiler-plugin.version> |
37 | 39 | <maven.minimum.version>3.3.1</maven.minimum.version> |
38 | 40 | <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version> |
39 | 41 | <maven-shade-plugin.version>2.4.3</maven-shade-plugin.version> |
| 42 | + <jcommon.version>1.0.23</jcommon.version> |
| 43 | + <jfreechart.version>1.0.13</jfreechart.version> |
40 | 44 | <logback.version>1.1.7</logback.version> |
41 | | - <scala.binary.version>2.11</scala.binary.version> |
42 | | - <spark.version>2.4.3</spark.version> |
| 45 | + <jcommander.version>1.27</jcommander.version> |
| 46 | + <azure.hadoop.version>2.7.4</azure.hadoop.version> |
| 47 | + <azure.storage.version>2.0.0</azure.storage.version> |
| 48 | + <spark.version>2.1.0</spark.version> |
| 49 | + <aws.sdk.version>1.11.109</aws.sdk.version> |
| 50 | + <jackson.version>2.5.1</jackson.version> |
| 51 | + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
43 | 52 | </properties> |
44 | 53 |
|
45 | | - <dependencyManagement> |
| 54 | + <dependencyManagement> |
46 | 55 | <dependencies> |
| 56 | + <dependency> |
| 57 | + <!-- Force Jackson versions to match Spark. This usually only matters for Spark local, not Spark submit --> |
| 58 | + <groupId>com.fasterxml.jackson.core</groupId> |
| 59 | + <artifactId>jackson-core</artifactId> |
| 60 | + <version>${jackson.version}</version> |
| 61 | + </dependency> |
| 62 | + <dependency> |
| 63 | + <groupId>com.fasterxml.jackson.core</groupId> |
| 64 | + <artifactId>jackson-databind</artifactId> |
| 65 | + <version>${jackson.version}</version> |
| 66 | + </dependency> |
| 67 | + <dependency> |
| 68 | + <groupId>com.fasterxml.jackson.core</groupId> |
| 69 | + <artifactId>jackson-annotations</artifactId> |
| 70 | + <version>${jackson.version}</version> |
| 71 | + </dependency> |
| 72 | + <dependency> |
| 73 | + <groupId>com.fasterxml.jackson.module</groupId> |
| 74 | + <artifactId>jackson-module-scala_${scala.binary.version}</artifactId> |
| 75 | + <version>${jackson.version}</version> |
| 76 | + </dependency> |
| 77 | + <dependency> |
| 78 | + <groupId>com.fasterxml.jackson.datatype</groupId> |
| 79 | + <artifactId>jackson-datatype-jdk8</artifactId> |
| 80 | + <version>${jackson.version}</version> |
| 81 | + </dependency> |
| 82 | + <dependency> |
| 83 | + <groupId>com.fasterxml.jackson.datatype</groupId> |
| 84 | + <artifactId>jackson-datatype-jsr310</artifactId> |
| 85 | + <version>${jackson.version}</version> |
| 86 | + </dependency> |
| 87 | + |
| 88 | + <dependency> |
| 89 | + <groupId>org.datavec</groupId> |
| 90 | + <artifactId>datavec-spark_${scala.binary.version}</artifactId> |
| 91 | + <version>1.0.0-beta7_spark_2</version> |
| 92 | + </dependency> |
| 93 | + <dependency> |
| 94 | + <groupId>org.deeplearning4j</groupId> |
| 95 | + <artifactId>deeplearning4j-ui</artifactId> |
| 96 | + <version>${dl4j-master.version}</version> |
| 97 | + </dependency> |
47 | 98 | <dependency> |
48 | 99 | <groupId>org.freemarker</groupId> |
49 | 100 | <artifactId>freemarker</artifactId> |
|
58 | 109 | </dependencyManagement> |
59 | 110 |
|
60 | 111 | <dependencies> |
| 112 | + <dependency> |
| 113 | + <groupId>org.apache.spark</groupId> |
| 114 | + <artifactId>spark-core_${scala.binary.version}</artifactId> |
| 115 | + <version>${spark.version}</version> |
| 116 | + </dependency> |
61 | 117 | <dependency> |
62 | 118 | <groupId>org.nd4j</groupId> |
63 | 119 | <artifactId>${nd4j.backend}</artifactId> |
64 | 120 | <version>${dl4j-master.version}</version> |
65 | 121 | </dependency> |
66 | 122 | <dependency> |
67 | 123 | <groupId>org.datavec</groupId> |
68 | | - <artifactId>datavec-api</artifactId> |
| 124 | + <artifactId>datavec-spark_${scala.binary.version}</artifactId> |
69 | 125 | <version>${dl4j-master.version}</version> |
70 | 126 | </dependency> |
71 | 127 | <dependency> |
72 | | - <groupId>org.datavec</groupId> |
73 | | - <artifactId>datavec-data-image</artifactId> |
| 128 | + <groupId>org.deeplearning4j</groupId> |
| 129 | + <artifactId>dl4j-spark_${scala.binary.version}</artifactId> |
74 | 130 | <version>${dl4j-master.version}</version> |
75 | 131 | </dependency> |
76 | 132 | <dependency> |
77 | | - <groupId>org.datavec</groupId> |
78 | | - <artifactId>datavec-spark_${scala.binary.version}</artifactId> |
79 | | - <version>1.0.0-SNAPSHOT</version> |
| 133 | + <groupId>org.deeplearning4j</groupId> |
| 134 | + <artifactId>dl4j-spark-parameterserver_${scala.binary.version}</artifactId> |
| 135 | + <version>${dl4j-master.version}</version> |
80 | 136 | </dependency> |
81 | 137 | <dependency> |
82 | | - <groupId>org.datavec</groupId> |
83 | | - <artifactId>datavec-local</artifactId> |
84 | | - <version>${dl4j-master.version}</version> |
| 138 | + <groupId>com.beust</groupId> |
| 139 | + <artifactId>jcommander</artifactId> |
| 140 | + <version>${jcommander.version}</version> |
85 | 141 | </dependency> |
| 142 | + <!-- Used for patent classification example --> |
86 | 143 | <dependency> |
87 | 144 | <groupId>org.deeplearning4j</groupId> |
88 | | - <artifactId>deeplearning4j-datasets</artifactId> |
| 145 | + <artifactId>deeplearning4j-nlp</artifactId> |
89 | 146 | <version>${dl4j-master.version}</version> |
90 | 147 | </dependency> |
91 | 148 | <dependency> |
92 | | - <groupId>org.deeplearning4j</groupId> |
93 | | - <artifactId>deeplearning4j-core</artifactId> |
| 149 | + <groupId>org.datavec</groupId> |
| 150 | + <artifactId>datavec-local</artifactId> |
94 | 151 | <version>${dl4j-master.version}</version> |
95 | 152 | </dependency> |
96 | 153 | <dependency> |
97 | 154 | <groupId>org.deeplearning4j</groupId> |
98 | | - <artifactId>deeplearning4j-ui</artifactId> |
| 155 | + <artifactId>deeplearning4j-zoo</artifactId> |
99 | 156 | <version>${dl4j-master.version}</version> |
100 | | - <exclusions> |
101 | | - <exclusion> |
102 | | - <groupId>net.jpountz.lz4</groupId> |
103 | | - <artifactId>lz4</artifactId> |
104 | | - </exclusion> |
105 | | - </exclusions> |
106 | 157 | </dependency> |
| 158 | + <!-- Used for patent classification example --> |
107 | 159 | <dependency> |
108 | | - <groupId>org.apache.spark</groupId> |
109 | | - <artifactId>spark-core_${scala.binary.version}</artifactId> |
110 | | - <version>${spark.version}</version> |
111 | | - <exclusions> |
112 | | - <exclusion> |
113 | | - <groupId>org.apache.hadoop</groupId> |
114 | | - <artifactId>log4j</artifactId> |
115 | | - </exclusion> |
116 | | - <exclusion> |
117 | | - <groupId>org.slf4j</groupId> |
118 | | - <artifactId>slf4j-log4j12</artifactId> |
119 | | - </exclusion> |
120 | | - </exclusions> |
| 160 | + <groupId>org.apache.hadoop</groupId> |
| 161 | + <artifactId>hadoop-azure</artifactId> |
| 162 | + <version>${azure.hadoop.version}</version> |
121 | 163 | </dependency> |
| 164 | + <!-- Used for patent classification example --> |
122 | 165 | <dependency> |
123 | | - <groupId>io.netty</groupId> |
124 | | - <artifactId>netty-all</artifactId> |
125 | | - <version>4.1.22.Final</version> |
| 166 | + <groupId>com.microsoft.azure</groupId> |
| 167 | + <artifactId>azure-storage</artifactId> |
| 168 | + <version>${azure.storage.version}</version> |
126 | 169 | </dependency> |
127 | 170 | <dependency> |
128 | | - <groupId>ch.qos.logback</groupId> |
129 | | - <artifactId>logback-classic</artifactId> |
130 | | - <version>${logback.version}</version> |
| 171 | + <groupId>com.amazonaws</groupId> |
| 172 | + <artifactId>aws-java-sdk-emr</artifactId> |
| 173 | + <version>${aws.sdk.version}</version> |
| 174 | + <scope>provided</scope> |
131 | 175 | </dependency> |
132 | 176 | <dependency> |
133 | | - <groupId>org.apache.httpcomponents</groupId> |
134 | | - <artifactId>httpclient</artifactId> |
135 | | - <version>4.3.5</version> |
| 177 | + <groupId>com.amazonaws</groupId> |
| 178 | + <artifactId>aws-java-sdk-s3</artifactId> |
| 179 | + <version>${aws.sdk.version}</version> |
| 180 | + <scope>provided</scope> |
| 181 | + </dependency> |
| 182 | + <dependency> |
| 183 | + <groupId>org.jsoup</groupId> |
| 184 | + <artifactId>jsoup</artifactId> |
| 185 | + <version>1.10.2</version> |
136 | 186 | </dependency> |
137 | 187 | </dependencies> |
138 | 188 |
|
139 | | - <!-- Maven Enforcer: Ensures user has an up to date version of Maven before building --> |
140 | 189 | <build> |
141 | 190 | <plugins> |
142 | | - <plugin> |
143 | | - <artifactId>maven-enforcer-plugin</artifactId> |
144 | | - <version>1.0.1</version> |
145 | | - <executions> |
146 | | - <execution> |
147 | | - <id>enforce-default</id> |
148 | | - <goals> |
149 | | - <goal>enforce</goal> |
150 | | - </goals> |
151 | | - <configuration> |
152 | | - <rules> |
153 | | - <requireMavenVersion> |
154 | | - <version>[${maven.minimum.version},)</version> |
155 | | - <message>********** Minimum Maven Version is ${maven.minimum.version}. Please upgrade Maven before continuing (run "mvn --version" to check). **********</message> |
156 | | - </requireMavenVersion> |
157 | | - </rules> |
158 | | - </configuration> |
159 | | - </execution> |
160 | | - </executions> |
161 | | - </plugin> |
162 | | - <plugin> |
163 | | - <groupId>org.apache.maven.plugins</groupId> |
164 | | - <artifactId>maven-compiler-plugin</artifactId> |
165 | | - <version>${maven-compiler-plugin.version}</version> |
166 | | - <configuration> |
167 | | - <source>${java.version}</source> |
168 | | - <target>${java.version}</target> |
169 | | - </configuration> |
170 | | - </plugin> |
171 | | - <plugin> |
172 | | - <groupId>com.lewisd</groupId> |
173 | | - <artifactId>lint-maven-plugin</artifactId> |
174 | | - <version>0.0.11</version> |
175 | | - <configuration> |
176 | | - <failOnViolation>true</failOnViolation> |
177 | | - <onlyRunRules> |
178 | | - <rule>DuplicateDep</rule> |
179 | | - <rule>RedundantPluginVersion</rule> |
180 | | - <!-- Rules incompatible with Java 9 |
181 | | - <rule>VersionProp</rule> |
182 | | - <rule>DotVersionProperty</rule> --> |
183 | | - </onlyRunRules> |
184 | | - </configuration> |
185 | | - <executions> |
186 | | - <execution> |
187 | | - <id>pom-lint</id> |
188 | | - <phase>validate</phase> |
189 | | - <goals> |
190 | | - <goal>check</goal> |
191 | | - </goals> |
192 | | - </execution> |
193 | | - </executions> |
194 | | - </plugin> |
195 | 191 | <plugin> |
196 | 192 | <groupId>org.codehaus.mojo</groupId> |
197 | 193 | <artifactId>exec-maven-plugin</artifactId> |
|
246 | 242 | </execution> |
247 | 243 | </executions> |
248 | 244 | </plugin> |
| 245 | + |
| 246 | + <plugin> |
| 247 | + <groupId>org.apache.maven.plugins</groupId> |
| 248 | + <artifactId>maven-compiler-plugin</artifactId> |
| 249 | + <version>3.5.1</version> |
| 250 | + <configuration> |
| 251 | + <source>${java.version}</source> |
| 252 | + <target>${java.version}</target> |
| 253 | + </configuration> |
| 254 | + </plugin> |
249 | 255 | </plugins> |
250 | | - <pluginManagement> |
251 | | - <plugins> |
252 | | - <plugin> |
253 | | - <groupId>org.eclipse.m2e</groupId> |
254 | | - <artifactId>lifecycle-mapping</artifactId> |
255 | | - <version>1.0.0</version> |
256 | | - <configuration> |
257 | | - <lifecycleMappingMetadata> |
258 | | - <pluginExecutions> |
259 | | - <pluginExecution> |
260 | | - <pluginExecutionFilter> |
261 | | - <groupId>com.lewisd</groupId> |
262 | | - <artifactId>lint-maven-plugin</artifactId> |
263 | | - <versionRange>[0.0.11,)</versionRange> |
264 | | - <goals> |
265 | | - <goal>check</goal> |
266 | | - </goals> |
267 | | - </pluginExecutionFilter> |
268 | | - <action> |
269 | | - <ignore/> |
270 | | - </action> |
271 | | - </pluginExecution> |
272 | | - </pluginExecutions> |
273 | | - </lifecycleMappingMetadata> |
274 | | - </configuration> |
275 | | - </plugin> |
276 | | - </plugins> |
277 | | - </pluginManagement> |
278 | 256 | </build> |
279 | 257 | </project> |
0 commit comments