install prerequisites
su
yum install openssl-devel
yum install zlib-devel
yum instal libffi-devel
install python 3
cd /app
wget https://www.python.org/ftp/python/3.9.0/Python-3.9.0.tgz
tar xzvf Python-3.9.0.tgz
cd Python-3.9.0
./configure
make test
make install
install python modules with pip3
pip3 install numpy
install libraries to connect to PostrgreSQL:
pip3 install psycopg2-binary
pip3 install graphviz
Sample web page connecting to PostgreSQL:
#!/usr/bin/python3
from datetime import datetime
import psycopg2
def getNavPane():
navPane=" <nav id=nav>"
navPane += "<div class=innertube>"
navPane +="<h3>Nodes manipulation</h3>"
navPane +="<ul> <li><a href=/cbaWorkflow/cbaWorkflow>Nodes list</a></li> <li><a href=/cbaWorkflow/addNodeForm>Add Node</a></li> <li><a href=/cbaWorkflow/addEdgeForm>Add dependency</a></li> </ul>"
navPane +=" <h3>Monitoring</h3> <ul> <li><a href=/cbaWorkflow/execStats>show execution statistics</a></li> <li><a href=/cbaWorkflow/cgi-bin/cadence>show execution graph</a></li>"
navPane +=" </ul> </div> </nav>"
return navPane;
print("Content-type: text/html")
print("")
print("")
print("<HTML>")
print("<HEAD><TITLE>Nodes list</TITLE>")
print("<link rel=stylesheet type=text/css href=/cbaWorkflow/css/cba.css>")
print("</HEAD>")
print("<BODY>")
print("<main><div class=\"innertube\">")
print("<H1>Nodes list</H1>")
now = datetime.now()
current_time = now.strftime("%m/%d/%Y at %H:%M:%S")
print("The list of nodes is generated on %s"%current_time)
print("<P>")
conn=None
try:
conn = psycopg2.connect(
host="192.168.2.30",
database="cbaworkflowdb",
user="dataexplorer1",
password="2MuchTime")
curr=conn.cursor()
strSQL="select * from nodes"
curr.execute(strSQL)
recordset=curr.fetchall()
print("<TABLE>")
print("<TR><TH>NODE ID</TH><TH>NODE NAME</TH></TR>")
for row in recordset:
print("<TR>")
print("<TD>%s</TD>"%row[0])
print("<TD>%s</TD>"%row[2])
print("</TR>")
curr.close()
print("</TABLE>")
except psycopg2.DatabaseError as e:
print(e)
finally:
if conn is not None:
conn.close()
print("</div></main>")
print(getNavPane())
print( "</BODY>")
print( "</HTML>")
Install and configure pyspark
pip3 install pyspark
in .bash_profile set the following:
export PYSPARK_PYTHON=/usr/local/bin/python3
export PYSPARK_DRIVER_PYTHON=python3