WORKDIR An error occurred while calling o64.parquet.
#DOWNLOAD SPARK WITH HADOOP INSTALL#
RUN conda install -quiet -y 'pyarrow' & \ SPARK_OPTS="-driver-java-options=-Xms1024M -driver-java-options=-Xmx4096M -driver-java-options=-Dlog4j.logLevel=info" \ RUN wget $ /usr/local/hadoopĮNV LD_LIBRARY_PATH $HADOOP_HOME/lib/nativeĮNV PATH $PATH:$SPARK_HOME/bin:$HADOOP_HOME/binĮNV SPARK_DIST_CLASSPATH $HADOOP_HOME/etc/hadoop:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/yarn:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*ĮNV SPARK_CLASSPATH $SPARK_HOME/jars/*:$SPARK_DIST_CLASSPĮNV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip \
#DOWNLOAD SPARK WITH HADOOP DOWNLOAD#
# Using the preferred mirror to download Spark SHELL Īpt-get install -no-install-recommends -y openjdk-8-jre-headless ca-certificates-java & \ # Distributed under the terms of the Modified BSD License.ĪRG BASE_CONTAINER=jupyter/scipy-notebook:54462805efcb # Copyright (c) Jupyter Development Team.
Val oldStrategy = (assemblyMergeStrategy in assembly).valueīase image : spoonshot/pyspark-notebook:2.4.7-hadoop-3.2-r1 Merge strategy for assembling conflictsĬase PathList("com", "esotericsoftware", xs _*) => MergeStrategy.lastĬase PathList("com", "squareup", xs _*) => MergeStrategy.lastĬase PathList("com", "sun", xs _*) => MergeStrategy.lastĬase PathList("com", "thoughtworks", xs _*) => MergeStrategy.lastĬase PathList("com", "google", xs _*) => MergeStrategy.lastĬase PathList("commons-beanutils", xs _*) => MergeStrategy.lastĬase PathList("commons-cli", xs _*) => MergeStrategy.lastĬase PathList("commons-collections", xs _*) => MergeStrategy.lastĬase PathList("commons-io", xs _*) => MergeStrategy.lastĬase PathList("io", "netty", xs _*) => MergeStrategy.lastĬase PathList("javax", "activation", xs _*) => MergeStrategy.lastĬase PathList("javax", "xml", xs _*) => MergeStrategy.lastĬase PathList("org", "apache", xs _*) => MergeStrategy.lastĬase PathList("org", "codehaus", xs _*) => MergeStrategy.lastĬase PathList("org", "fusesource", xs _*) => MergeStrategy.lastĬase PathList("org", "mortbay", xs _*) => MergeStrategy.lastĬase PathList("org", "tukaani", xs _*) => MergeStrategy.lastĬase PathList("org", "tensorflow", xs _*) => MergeStrategy.lastĬase PathList("javax", "annotation", xs _*) => MergeStrategy.lastĬase PathList("com", "amazonaws", xs _*) => MergeStrategy.lastĬase PathList("xerces", xs _*) => MergeStrategy.lastĬase PathList("xmlenc", xs _*) => MergeStrategy.lastĬase "about.html" => MergeStrategy.renameĬase "META-INF/ECLIPSEF.RSA" => MergeStrategy.lastĬase "META-INF/mailcap" => MergeStrategy.lastĬase "META-INF/fault" => MergeStrategy.lastĬase "plugin.properties" => MergeStrategy.lastĬase "log4j.properties" => MergeStrategy.last LibraryDependencies += "com.amazonaws" % "aws-java-sdk-s3" % "1.11.603" % "provided"ĪssemblyJarName in assembly := s"base-assembly.jar" LibraryDependencies += "" %% "mongo-spark-connector" % "2.4.2"
LibraryDependencies += "mysql" % "mysql-connector-java" % "8.0.12" LibraryDependencies += "" %% "spark-core" % "2.4.7" % "provided"
0 kommentar(er)
