PySpark：无法创建火花数据帧

我是Python/Spark的新手。我试图创建一个从熊猫数据帧的火花数据框，但得到以下错误：PySpark：无法创建火花数据帧

from pyspark import SparkContext 
from pyspark.sql import SQLContext 
from pyspark.sql import HiveContext 
sqlctx = SQLContext(sc) 

type(sqlctx) 

pyspark.sql.context.SQLContext 
    --------------------------------------------------------------------------- 
Py4JJavaError        Traceback (most recent call last) 
<ipython-input-30-c0ccd84867c3> in <module>() 
----> 1 train_sparkdf = sqlctx.createDataFrame(train_ml) 

C:\spark-2.0.2-bin-hadoop2.7\python\pyspark\sql\context.pyc in createDataFrame(self, data, schema, samplingRatio, verifySchema) 
    307   Py4JJavaError: ... 
    308   """ 
--> 309   return self.sparkSession.createDataFrame(data, schema, samplingRatio, verifySchema) 
    310 
    311  @since(1.3) 

C:\spark-2.0.2-bin-hadoop2.7\python\pyspark\sql\session.pyc in createDataFrame(self, data, schema, samplingRatio, verifySchema) 
    524    rdd, schema = self._createFromLocal(map(prepare, data), schema) 
    525   jrdd = self._jvm.SerDeUtil.toJavaArray(rdd._to_java_object_rdd()) 
--> 526   jdf = self._jsparkSession.applySchemaToPythonRDD(jrdd.rdd(), schema.json()) 
    527   df = DataFrame(jdf, self._wrapped) 
    528   df._schema = schema 

C:\spark-2.0.2-bin-hadoop2.7\python\lib\py4j-0.10.3-src.zip\py4j\java_gateway.py in __call__(self, *args) 
    1131   answer = self.gateway_client.send_command(command) 
    1132   return_value = get_return_value(
-> 1133    answer, self.gateway_client, self.target_id, self.name) 
    1134 
    1135   for temp_arg in temp_args: 

C:\spark-2.0.2-bin-hadoop2.7\python\pyspark\sql\utils.pyc in deco(*a, **kw) 
    61  def deco(*a, **kw): 
    62   try: 
---> 63    return f(*a, **kw) 
    64   except py4j.protocol.Py4JJavaError as e: 
    65    s = e.java_exception.toString() 

C:\spark-2.0.2-bin-hadoop2.7\python\lib\py4j-0.10.3-src.zip\py4j\protocol.py in get_return_value(answer, gateway_client, target_id, name) 
    317     raise Py4JJavaError(
    318      "An error occurred while calling {0}{1}{2}.\n". 
--> 319      format(target_id, ".", name), value) 
    320    else: 
    321     raise Py4JError(

Py4JJavaError: An error occurred while calling o28.applySchemaToPythonRDD. 
: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient 
    at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522) 
    at org.apache.spark.sql.hive.client.HiveClientImpl.<init>(HiveClientImpl.scala:189) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) 
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) 
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423) 
    at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:258) 
    at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:359) 
    at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:263) 
    at org.apache.spark.sql.hive.HiveSharedState.metadataHive$lzycompute(HiveSharedState.scala:39) 
    at org.apache.spark.sql.hive.HiveSharedState.metadataHive(HiveSharedState.scala:38) 
    at org.apache.spark.sql.hive.HiveSharedState.externalCatalog$lzycompute(HiveSharedState.scala:46) 
    at org.apache.spark.sql.hive.HiveSharedState.externalCatalog(HiveSharedState.scala:45) 
    at org.apache.spark.sql.hive.HiveSessionState.catalog$lzycompute(HiveSessionState.scala:50) 
    at org.apache.spark.sql.hive.HiveSessionState.catalog(HiveSessionState.scala:48) 
    at org.apache.spark.sql.hive.HiveSessionState$$anon$1.<init>(HiveSessionState.scala:63) 
    at org.apache.spark.sql.hive.HiveSessionState.analyzer$lzycompute(HiveSessionState.scala:63) 
    at org.apache.spark.sql.hive.HiveSessionState.analyzer(HiveSessionState.scala:62) 
    at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:49) 
    at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) 
    at org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:666) 
    at org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:656) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 
    at java.lang.reflect.Method.invoke(Method.java:498) 
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237) 
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) 
    at py4j.Gateway.invoke(Gateway.java:280) 
    at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) 
    at py4j.commands.CallCommand.execute(CallCommand.java:79) 
    at py4j.GatewayConnection.run(GatewayConnection.java:214) 
    at java.lang.Thread.run(Thread.java:745) 
Caused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient 
    at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1523) 
    at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:86) 
    at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) 
    at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) 
    at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) 
    at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) 
    at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) 
    ... 32 more 
Caused by: java.lang.reflect.InvocationTargetException 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) 
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) 
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423) 
    at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) 
    ... 38 more 
Caused by: javax.jdo.JDOFatalInternalException: Error creating transactional connection factory 
NestedThrowables: 
java.lang.reflect.InvocationTargetException 
    at org.datanucleus.api.jdo.NucleusJDOHelper.getJDOExceptionForNucleusException(NucleusJDOHelper.java:587) 
    at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:788) 
    at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) 
    at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 
    at java.lang.reflect.Method.invoke(Method.java:498) 
    at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) 
    at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) 
    at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) 
    at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) 
    at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) 
    at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) 
    at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) 
    at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) 
    at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76) 
    at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136) 
    at org.apache.hadoop.hive.metastore.RawStoreProxy.<init>(RawStoreProxy.java:57) 
    at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) 
    at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:66) 
    at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) 
    at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:199) 
    at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:74) 
    ... 43 more 
Caused by: java.lang.reflect.InvocationTargetException 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) 
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) 
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423) 
    at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) 
    at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:325) 
    at org.datanucleus.store.AbstractStoreManager.registerConnectionFactory(AbstractStoreManager.java:282) 
    at org.datanucleus.store.AbstractStoreManager.<init>(AbstractStoreManager.java:240) 
    at org.datanucleus.store.rdbms.RDBMSStoreManager.<init>(RDBMSStoreManager.java:286) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) 
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) 
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423) 
    at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) 
    at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) 
    at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) 
    at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) 
    at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) 
    ... 72 more 
Caused by: org.datanucleus.exceptions.NucleusException: Attempt to invoke the "BONECP" plugin to create a ConnectionPool gave an error : The specified datastore driver ("com.mysql.jdbc.Driver") was not found in the CLASSPATH. Please check your CLASSPATH specification, and the name of the driver. 
    at org.datanucleus.store.rdbms.ConnectionFactoryImpl.generateDataSources(ConnectionFactoryImpl.java:259) 
    at org.datanucleus.store.rdbms.ConnectionFactoryImpl.initialiseDataSources(ConnectionFactoryImpl.java:131) 
    at org.datanucleus.store.rdbms.ConnectionFactoryImpl.<init>(ConnectionFactoryImpl.java:85) 
    ... 90 more 
Caused by: org.datanucleus.store.rdbms.connectionpool.DatastoreDriverNotFoundException: The specified datastore driver ("com.mysql.jdbc.Driver") was not found in the CLASSPATH. Please check your CLASSPATH specification, and the name of the driver. 
    at org.datanucleus.store.rdbms.connectionpool.AbstractConnectionPoolFactory.loadDriver(AbstractConnectionPoolFactory.java:58) 
    at org.datanucleus.store.rdbms.connectionpool.BoneCPConnectionPoolFactory.createConnectionPool(BoneCPConnectionPoolFactory.java:54) 
    at org.datanucleus.store.rdbms.ConnectionFactoryImpl.generateDataSources(ConnectionFactoryImpl.java:238) 
    ... 92 more

类似的问题以前问，我已经尝试了所有的解决方案，但徒劳无功。任何人都可以帮我设置并在我的机器上成功运行spark吗？

版本： Jupyter：4.2.3 的Python：2.7.12 星火：2.0.2（Hadoop的2.7）

来源

2016-12-17 avinax

你是否在本地模式或纱线使用火花？你在Spark的conf /'目录下有'hive-site.xml'文件吗？也许你已经在'conf/spark-defaults.conf'中配置了一些自定义配置单元Metastore配置？ – Mariusz

翻翻你的错误消息的链，它们表明您已经配置了蜂巢metastore到使用mysql JDBC驱动程序，但在类路径中找不到该驱动程序。

“vanilla”Spark设置应该使用Hive Metastore的内置derby JDBC驱动程序。从你的问题来看，你看起来正处于尝试基本事物的阶段，所以我重新配置Hive来使用mysql并不是有意的，你可能需要摆脱它。不是一个解决方案，但至少有一个指针在哪里看得更远...

来源

2016-12-19 09:47:59 okoethibm

您好Mariusz和okothibm，我在conf /目录中有一个hive-site.xml文件，并且我从这个链接[hive-site.xml]下载了它（https://github.com/apache/hive/blob/主/数据/ CONF /蜂巢-site.xml中）。你认为我需要更改设置吗？我怎样才能改变它在本地机器上运行它？ – avinax

PySpark：无法创建火花数据帧

回答

相关问题