I am trying to migrate PySpark code into Python pandas lib.
Mainly facing issue with configParser() , as Spark use SparkConf/SparkContext:
Pyspark
def __init__(self):
'''
Constructor
'''
self.currentYear = datetime.datetime.now().year
base_path = os.environ['BASE_PATH']
self.config = StringIO.StringIO()
self.configParser = ConfigParser.ConfigParser()
# Read the property file messageResource.properties and store it in baseProp
print "reading messageResource.properties"
self.config.write('[msgProp]\n')
self.config.write(open(base_path+'/conf/messageResource.properties').read())
print "reading app.properties"
self.config.write('[appProp]\n')
self.config.write(open(base_path+'/conf/app.properties').read())
self.config.seek(0, os.SEEK_SET)
self.configParser.readfp(self.config)
print "done reading property files"
...
Python
def __init__(self):
self.currentYear = datetime.datetime.now().year
#base_path = os.environ['BASE_PATH']
#base_path="./conf"
aca_base_path = f'C:\Repo\reports-pyspark-scripts\reports-pyscripts\conf'
self.config_parser = configparser.ConfigParser()
print("reading messageResource.properties")
self.config_parser.read(os.path.join(base_path, 'conf', 'messageResource.properties'))
print("reading app.properties")
self.config_parser.read(os.path.join(base_path, 'conf', 'app.properties'))
print("done reading property files")
def get_all_property_msgs(self):
self.error_client_id_key = self.config_parser.get('msgProp', 'error_clientId_key')
Getting an error :
self.error_client_id_key = self.config_parser.get('msgProp', 'error_clientId_key')
raise NoSectionError(section) from None
configparser.NoSectionError: No section: 'msgProp'
The error is telling you that the config is broken, not configparser. The section you expect in the config is not there.
@MisterMiyagi Could you pls elaborate, i had the existing code in pyspark and i am trying to just work around that, the config is just 2 properties files as mention in snippet