memberName=OCC
dateRange=-30:7
year=2017
month=10
hiveSchema=occdata
job.env=prod
jdbcDriver = oracle.jdbc.driver.OracleDriver
jdbcUrl= jdbc:oracle:thin:navistar
jdbcSchema=QARES1
hdfsUrl =hdfs://prodnameservice1/user/yyy1k78/
temp=/users/yyy1k78
output=OCCDataPoint_Mapping
input=OCCDataPoint_Input
numPartitions =50
#format = com.databricks.spark.avro
format=parquet
compressionCodecName =org.apache.hadoop.io.compress.snappyCodec
fetchSize = 500
#************* Spark Config Properties ***************************#
spark.app.name = OCCDataPoint_Mapping
#spark.master = yarn
#spark.storage.memoryFraction=0.4
spark.sql.shuffle.partitions=2000
spark.shuffle.fraction=0.45
spark.default.parallelism=100
#spark.eventLog.enabled=false
#spark.scheduler.revive.interval=1s
spark.driver.memory=8g
#************* Hadoop Properties **********************************#
hadoop.security.authentication=kerberos
spark.executor.memory=16g
spark.yarn.executor.memoryOverhead=8192
#spark.yarn.am.cores=2
spark.driver.am.memory=8G
spark.yarn.am.memoryOverhead=8g
#spark.local.dir=/tmp
#spark.logConf=false
spark.scheduler.mode=FAIR
#spark.checkpoint.dir=/tmp/datapoint
###SHUFFLE MODE####
#spark.reducer.maxSizeInFlight=48m
spark.shuffle.compress=true
#spark.shuffle.file.buffer=32k
#spark.shuffle.io.maxRetries= 3
#spark.shuffle.io.numConnectionsPerPeer= 1
#spark.shuffle.manager=sort
###This must be enabled if spark.dynamicAllocation.enabled is true. The external shuffle service must be set up in order to enable it.
#spark.shuffle.service.port=7337
#spark.shuffle.sort.bypassMergeThreshold=200
spark.shuffle.spill.compress=true
#spark.shuffle.service.enabled=true
###COMPRESSION AND SERIALIZATION####
spark.broadcast.compress=true
spark.io.compression.codec=snappy
spark.dynamicAllocation.enabled=true

## HIVE JDBC ######################################
java.security.krb5.conf=krb5.conf
javax.security.auth.useSubjectCredsOnly=true
hive.jdbc.url=jdbc:hive2://brksvl168:10000/occdata;principal=hive/_HOST@AD.NAVISTAR.COM;ssl=true
hive.jdbc.driver=org.apache.hive.jdbc.HiveDriver
keytab.file=va_dflt.keytab

## FCAP ####
fcap.job.output.hdfs.parts.path=hdfs://prodnameservice1/user/yyy1k78/fcap/parts
fcap.job.output.hdfs.opcode.path=hdfs://prodnameservice1/user/yyy1k78/fcap/opcodes
fcap.hive.schema=fcap
fcap.hive.parts.table=fcap_parts
fcap.hive.opcodes.table=fcap_opcodes
hive.optimize.reducededuplication.min.reducer=4
hive.optimize.reducededuplication=true
hive.merge.mapfiles=true
hive.merge.mapredfiles=false
hive.merge.smallfiles.avgsize=16000000
hive.merge.size.per.task=256000000
hive.merge.sparkfiles=true
hive.auto.convert.join=true
hive.auto.convert.join.noconditionaltask=true
hive.auto.convert.join.noconditionaltask.size=20M(might need to increase for Spark, 200M)
hive.optimize.bucketmapjoin.sortedmerge=false
hive.map.aggr.hash.percentmemory=0.5
hive.map.aggr=true
hive.optimize.sort.dynamic.partition=false
hive.stats.autogather=true
hive.stats.fetch.column.stats=true
hive.compute.query.using.stats=true
hive.limit.pushdown.memory.usage=0.4 (MR and Spark)
hive.optimize.index.filter=true
hive.exec.reducers.bytes.per.reducer=67108864
hive.smbjoin.cache.rows=10000
hive.fetch.task.conversion=more
hive.fetch.task.conversion.threshold=1073741824
hive.optimize.ppd=true
###############################################
parquet.enable.summary-metadata=false
spark.sql.parquet.binaryAsString=true
spark.sql.parquet.mergeSchema=true
spark.sql.parquet.compression.codec=snappy
spark.rdd.compress=true
spark.io.compression.codec=snappy
spark.sql.tungsten.enabled=false
spark.sql.codegen=false
spark.sql.unsafe.enabled=false
######### GEOJSON FILES ######################
RoadsFilePath=/user/yyy1k78/occtelematics/data/roadsgeojson
RampsFilePath=/user/yyy1k78/occtelematics/data/rampsgeojson
DecelFilePath=/user/yyy1k78/occtelematics/data/decelgeojson
TurnsFilePath=/user/yyy1k78/occtelematics/data/turnsgeojson
IRIFilePath=/user/yyy1k78/occtelematics/data/irigeojson
###### Frequent Pattern FAULTS ##############
min.confidence=0.001
min.support=0.1
index=15
