TDSS YARN Integration Guide
  • Introduction
  • Preqrequisites
  • TDSS Hadoop Artefacts setup
  • Important YARN entries in dss.properties
  • DB Entries
  • Update the api war file
Powered by GitBook
On this page

Important YARN entries in dss.properties

PreviousTDSS Hadoop Artefacts setupNextDB Entries

Last updated 7 years ago

# SPECIFY CORRECT JAVA_HOME

yarn.mode.java.home=/usr/jdk64/jdk1.8.0_60/

yarn.mode.java.xmx=-Xmx2048m

# SPECIFY QUEUE (Check correct queue, if you are using your own hadoop, use default)

yarn.mode.queue.name=default

# SPECIFY CONTAINER AND SPARK RESOURCES

yarn.mode.container.resource.memory=2500

yarn.mode.container.resource.cores=1

spark.driver.memory=1g

spark.driver.cores=1

spark.executor.memory=2g

spark.executor.cores=1

spark.executor.instances=1

# SPECIFY NAME NODE AND RESOURCE MANAGER CONTEXT

yarn.mode.name.node.root.context=hdfs://hklpadhas01.hk.standardchartered.com:8020

yarn.mode.resource.manager.context=

# SPECIFY user.name FOR SECURE ACCESS

yarn.api.additional.request.params=?user.name=ec2-user

# SPECIFY CORRECT PATHS FOR TDSS LIBS / SPARK FILES / UDF FILES IN HADOOP

yarn.mode.app.jar.path=<base dir>/tdss/lib

yarn.mode.other.lib.path=<base dir>/tdss/spark

yarn.mode.spark.lib.path=<base dir>/tdss/spark/spark-assembly-1.6.1-hadoop2.6.0.jar

yarn.mode.udf.dep.path=<base dir>/tdss/udf

# SPECIFY correct egg file for db access (MySQL / jaydebeapi etc)

yarn.mode.db.egg.file=JayDeBeApi-1.1.1-py2.7.egg

# CLASS PATH ENTRIES. Verify hadoop lib paths from admin managing hadoop cluster. Below replace /home/ownappsg with correct hadoop path.

yarn.mode.hadoop.home=/home/ownappsg/hadoop/etc/hadoop/share/hadoop/common/*<CPS>/home/ownappsg/hadoop/etc/hadoop/share/hadoop/common/lib/*<CPS>/home/ownappsg/hadoop/etc/hadoop/share/hadoop/hdfs/*<CPS>/home/ownappsg/hadoop/etc/hadoop/share/hadoop/hdfs/lib/*<CPS>/home/ownappsg/hadoop/etc/hadoop/share/hadoop/yarn/*<CPS>/home/ownappsg/hadoop/etc/hadoop/share/hadoop/yarn/lib/*<CPS>/home/ownappsg/hadoop/etc/hadoop/share/hadoop/mapreduce/*<CPS>/home/ownappsg/hadoop/etc/hadoop/share/hadoop/mapreduce/lib/*

yarn.mode.hadoop.conf=/home/ownapsg/hadoop/etc/hadoop/conf

# SPECIFY correct hadoop version

yarn.mode.hadoop.version=2.6.5

# SPECIFY user used to launch yarn jobs

yarn.mode.hadoop.user.name=ownappsg

# BELOW THIS COPY PROPERTIES GENERATED BY RUNNING get_hdfs_files_timestamp_size.sh above OUTPUT TO tdss_properties

http://hklpadhas02.hk.standardchartered.com:8050