- Get project id
-
CEL expression:
ctx.scope.id
REST API:
In this example the name of the project is "project1." Replace with the name of your
project.
$ project=`curl --request GET "${CPD_URL}/v2/projects?name=project1" -H "Authorization: Bearer ${bearer_token}" -k -s`
$ proj_id=`echo $project | jq -r '.resources|.[0].metadata.guid'`
$ echo $proj_id
-
dsjob CLI:
$ project=`cpdctl dsjob list-projects --with-id |grep project1`
$ proj_id=`echo ${project#*|}`
$ echo $proj_id
- Get job id
-
CEL expression:
ctx.job.id
REST
API:
$ joblist=`curl --request GET "${CPD_URL}/v2/jobs?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}" -k -s`
$ echo $joblist |jq
This command prints a list of jobs. Replace "testjob" with the name of
your job in the following
commands.
$ echo $joblist|jq -r '.results[].metadata|select(.name=="testjob"?) |.asset_id'
-
dsjob CLI:
$ job=`cpdctl dsjob list-jobs --project-id ${proj_id} --with-id | grep testjob`
$ echo $job
testjob |317dce0d-247a-48aa-bd96-f4ec44cb39fc
$ job_id=`echo ${job#*|}`
$ echo $job_id
- Get job run id
-
CEL expression:
ctx.job_run.id
REST
API:
$ runlist=`curl --request GET "${CPD_URL}/v2/jobs/${job_id}/runs?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}" -k -s`
$ echo $runlist
This prints a list of job runs. Select a run based on description, create time,
or other
metadata.
$ echo $runlist|jq -r '.results[]|select(.metadata.description=="Initial run"?)
-
dsjob CLI:
$ cpdctl dsjob list-jobruns --project-id ${proj_id} --id $job_id --detail
- DSAddEnvVar
- REST API:
#!/bin/sh
# param1: environment name
# param2: new variable name
# param3: new variable value
DSAddEnvVar()
{
envs=`curl -k -s --request GET "${CPD_URL}/v2/environments?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
env_id=`echo $envs|jq -r ".resources[] | select(.metadata.name==\"$1\") | .metadata.asset_id"`
existingvars=`echo $envs|jq -r ".resources[] | select(.metadata.name==\"$env_name\") | .entity.environment.environment_variables"`
updatevars=`echo $existingvars |jq ".$2 = \"$3\""`
data="{\"/entity/environment/environment_variables\": ${updatevars}}"
curl -k -s --request PATCH "${CPD_URL}/v2/environments/${env_id}?project_id=${proj_id}&environment_guid=${proj_id}" --header 'Content-Type: application/json' --header "Authorization: Bearer ${bearer_token}" --data-raw "${data}"
}
- dsjob CLI:
#!/bin/sh
# param1: environment name
# param2: new variable name
# param3: new variable value
DSAddEnvVar()
{
cpdctl dsjob update-env-vars --project-id ${proj_id} -n $1 --env $2=$3
}
The following syntax adds a new environment variable to the specified environment.
$ DSAddEnvVar "default_datastage_px" "newvar" "val"
- DSAddProject
- REST API:
#!/bin/sh
DSAddProject()
{
data="{\"name\": \"$1\", \"generator\": \"customer\", \"storage\": {\"type\": \"assetfiles\"}}"
proj=`curl -k -s --request POST "${CPD_URL}/transactional/v2/projects" --header "Authorization: Bearer ${bearer_token}" -d "$data" --header 'Content-Type: application/json'`
echo $proj|jq -r ".location" |awk -F '/' '{print $4}'
}
- dsjob CLI:
#!/bin/sh
DSAddProject()
{
proj_id=`cpdctl dsjob create-project -n "$1"`
echo $proj_id
}
The following syntax creates a new project and retrieves its
id.$ proj_id=`DSAddProject "demo-project"`
- DSDeleteEnvVar
- REST API:
#!/bin/sh
# param1: environment name
# param2: variable name
DSDeleteEnvVar()
{
envs=`curl -k -s --request GET "${CPD_URL}/v2/environments?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
env_id=`echo $envs|jq -r ".resources[] | select(.metadata.name==\"$1\") | .metadata.asset_id"`
existingvars=`echo $envs|jq -r ".resources[] | select(.metadata.name==\"$env_name\") | .entity.environment.environment_variables"`
updatevars=`echo $existingvars |jq "del(.$2)"`
data="{\"/entity/environment/environment_variables\": ${updatevars}}"
curl -k -s --request PATCH "${CPD_URL}/v2/environments/${env_id}?project_id=${proj_id}&environment_guid=${proj_id}" --header 'Content-Type: application/json' --header "Authorization: Bearer ${bearer_token}" --data-raw "${data}"
}
- dsjob CLI:
#!/bin/sh
# param1: environment name
# param2: variable name
DSDeleteEnvVar()
{
cpdctl dsjob delete-env-vars --project-id ${proj_id} -n $1 --env $2
}
The following syntax deletes a new environment variable from the specified
environment.
$ DSDeleteEnvVar "default_datastage_px" "uservar1"
- DSDeleteProject
- REST API:
#!/bin/sh
DSDeleteProject()
{
curl -k -s --request DELETE "${CPD_URL}/transactional/v2/projects/$1?project_id=$1" --header "Authorization: Bearer ${bearer_token}"
}
- The following syntax deletes a project. The project id is
required.
$ DSDeleteProject "${proj_id}"
- DSExecute
- Define a wrapper function.
#!/bin/sh
DSExecute()
{
shelltype=$1
command=$2
output=$3
sh -c $command > $output 2>&1
}
Call the function.
$ DSExecute "UNIX" "echo someinfo" "output.log"
$ returnCode=$?
- DSGetIDForJob
- REST API:
DSGetIdForJob()
{
joblist=`curl --request GET "${CPD_URL}/v2/jobs?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}" -k -s`
echo $joblist|jq -r ".results[].metadata|select(.name==\"$1\") |.asset_id"
}
- dsjob CLI:
DSGetIdForJob()
{
job=`cpdctl dsjob list-jobs --project-id ${proj_id} --with-id | grep $1`
echo ${job#*|}
}
The following syntax retrieves job ID by job
name.$ job_id=`DSGetIdForJob "testjob"``
- DSGetJobInfo
- You can rewrite the BASIC function DSGetJobInfo as a wrapper function that uses the REST API
with the following syntax. The job_id, run_id, and project_id are required to retrieve the job
information. Some information, such as DSJ.JOBSTARTTIMESTAMP, is not supported.
#!/bin/sh
DSGetJobInfo()
{
# Get the job run metadata
jobrun=`curl --request GET "${CPD_URL}/v2/jobs/${job_id}/runs/${run_id}?job_id=${job_id}&run_id=${run_id}&project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
# Parse and extract the information
if [ "$1" = "DSJ.JOBSTATUS" ]; then
echo $jobrun|jq -r '.entity.job_run.state'
elif [ "$1" = "DSJ.JOBNAME" ]; then
echo $jobrun|jq -r '.entity.job_run.job_name'
elif [ "$1" = "DSJ.PARAMLIST" ]; then
echo $jobrun|jq -r '.entity.job_run.configuration.job_parameters[]'
elif [ "$1" = "DSJ.STAGELIST" ] || [$1 = "DSJ.FULLSTAGELIST"]; then
echo $jobrun|jq -r '.entity.job_run.configuration.metrics.stage_metrics[].stage_name'
elif [ "$1" = "DSJ.USERSTATUS" ]; then
echo $jobrun|jq -r '.entity.job_run.configuration.userStatus'
elif [ "$1" = "DSJ.JOBDESC" ] || ["$1" = "DSJ.JOBFULLDESC"]; then
echo $jobrun|jq -r '.metadata.description'
elif [ "$1" = "DSJ.JOBELAPSED" ]; then
echo $jobrun|jq -r '.entity.job_run.duration'
fi
}
- You can also rewrite DSGetJobInfo as a wrapper function that uses the djsob CLI with the
following syntax.
#!/bin/sh
DSGetJobInfo()
{
# Get the job run metadata
jobrun=`cpdctl dsjob jobrunstat --project-id ${proj_id} --id ${job_id} --run-id ${run_id} --output json --with-metadata | sed '$d'`
# Parse and extract the information
if [ "$1" = "DSJ.JOBSTATUS" ]; then
echo $jobrun|jq -r '.entity.job_run.state'
elif [ "$1" = "DSJ.JOBNAME" ]; then
echo $jobrun|jq -r '.entity.job_run.job_name'
elif [ "$1" = "DSJ.PARAMLIST" ]; then
echo $jobrun|jq -r '.entity.job_run.configuration.job_parameters[]'
elif [ "$1" = "DSJ.STAGELIST" ] || [$1 = "DSJ.FULLSTAGELIST"]; then
echo $jobrun|jq -r '.entity.job_run.configuration.metrics.stage_metrics[].stage_name'
elif [ "$1" = "DSJ.USERSTATUS" ]; then
echo $jobrun|jq -r '.entity.job_run.configuration.userStatus'
elif [ "$1" = "DSJ.JOBDESC" ] || ["$1" = "DSJ.JOBFULLDESC"]; then
echo $jobrun|jq -r '.metadata.description'
elif [ "$1" = "DSJ.JOBELAPSED" ]; then
echo $jobrun|jq -r '.entity.job_run.duration'
fi
}
The following syntax retrieves the status of the job.
$ jobstatus=`DSGetJobInfo "DSJ.JOBSTATUS"`
- DSGetUserStatus
- To get the user status of a DataStage job in the script of a Run Bash script node, get the
project id (
ctx.scope.id
). If the Run DataStage flow is named
run_datastage_flow
, the job id is
tasks.run_datastage_flow.results.job
and the run id is
tasks.run_datastage_flow.results.job_run
. The following syntax uses these
parameters to retrieve the user status.export CPDCTL_ENABLE_DSJOB=1
DSGetUserStatus()
{
# get last 36 characters of output string, which holds the job id or job run id of datastage flow
flowid=${ds_job_id:0-36:36}
flowrunid=${ds_job_run_id:0-36:36}
# get jobrun metadata by dsjob command or REST API
jobrun=`cpdctl dsjob get-jobrun --project-id $project_id --id $flowid --run-id $flowrunid --output json --with-metadata`
# extract userStatus message from job run
echo $jobrun | jq -r '.entity.job_run.configuration.userStatus'
}
userstatus=`DSGetUserStatus`
- DSGetLinkInfo
- To retrieve the name of a link and the number of rows that have passed down a link,
get the project id, job id, run id, and link name. You can rewrite DSGetLinkInfo into a
wrapper function using the following syntax.
- REST API:
-
#!/bin/sh
DSGetLinkInfo()
{
# Get the job run metadata
jobrun=`curl -k -s --request GET "${CPD_URL}/v2/jobs/${job_id}/runs/${run_id}?job_id=${job_id}&run_id=${run_id}&project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
# Parse and extract the information
if [ "$1" = "DSJ.LINKNAME" ]; then
echo $jobrun|jq -r ".entity.job_run.configuration.metrics.link_metrics[]|select(.link_name==\"$2\"?) |.link_name"
elif [ "$1" = "DSJ.LINKROWCOUNT" ]; then
echo $jobrun|jq -r ".entity.job_run.configuration.metrics.link_metrics[]|select(.link_name==\"$2\"?) |.rows_written"
fi
}
- dsjob CLI:
#!/bin/sh
DSGetLinkInfo()
{
# Get the job run metadata
jobrun=`cpdctl dsjob get-jobrun --project-id ${proj_id} --id ${job_id} --run-id ${run_id} --output json`
# Parse and extract the information
if [ "$1" = "DSJ.LINKNAME" ]; then
echo $jobrun|jq -r ".job_run.configuration.metrics.link_metrics[]|select(.link_name==\"$2\"?) |.link_name"
elif [ "$1" = "DSJ.LINKROWCOUNT" ]; then
echo $jobrun|jq -r ".job_run.configuration.metrics.link_metrics[]|select(.link_name==\"$2\"?) |.rows_written"
fi
}
The following function call gets the link row count for
Link_1.$ rows=`DSGetLinkInfo "DSJ.LINKROWCOUNT" "Link_1"`
- DSGetLogEntry, DSGetLogEntryFull, DSGetLogEventIds, DSGetLogSummary,
DSGetNewestLogId
- To get the logs of a job run, get the project id, job id, and run id. You can write wrapper
functions to retrieve log IDs, event IDs, entries, and summaries with the following syntax.
- REST API:
-
#!/bin/sh
GetAllLogs()
{
logs=`curl -k -s --request GET "${CPD_URL}/v2/jobs/${job_id}/runs/${run_id}/logs?job_id=${job_id}&run_id=${run_id}&project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
logs_strip=${logs#*results\":\[\"}
logs_strip=${logs_strip%\"\],\"total_count*}
logs_array=${logs_strip//\\\"/\"}
logs_array=${logs_array//\\\\\"/\\\"}
logs_array=${logs_array//\\\\n/\\n}
echo $logs_array | jq
}
DSGetLogEventIds()
{
# Get the job run logs metadata
logs_array=`GetAllLogs`
# Parse and extract the information
idlist=`echo $logs_array | jq -r '.[].eventID'`
echo $idlist
}
DSGetNewestLogId()
{
# Get the job run logs metadata
logs_array=`GetAllLogs`
# Parse and extract the information
echo $logs_array | jq -r "max_by(.eventID|tonumber)|.eventID"
}
DSGetLogEntry()
{
# Get the job run logs metadata
logs_array=`GetAllLogs`
# Parse and extract the information
echo $logs_array | jq -r ".[]|select(.eventID==\"$1\")|[.occurredAt,.user,.eventType,.eventDetail]|join(\"\\\\\")"
}
DSGetLogSummary()
{
# Get the job run logs metadata
logs_array=`GetAllLogs`
# Parse and extract the information
echo $logs_array | jq -r ".[]|[.eventID,.eventType,(.eventDetail|split(\"\n\")[0])]|join(\"\\\\\")"
}
- dsjob CLI:
DSGetLogEventIds()
{
logsum=`cpdctl dsjob logsum --project-id $proj_id --job-id $job_id --run-id $run_id`
echo $logsum | awk '{print $3}' | sed '$d'
}
DSGetNewestLogId()
{
logsum=`cpdctl dsjob logsum --project-id $proj_id --job-id $job_id --run-id $run_id`
echo $logsum | awk '{print $3}' | tac | sed '1,2d' | head -n 1
}
DSGetLogEntry()
{
entry=`cpdctl dsjob logdetail --project-id $proj_id --job-id $job_id --run-id $run_id --eventrange $1-$1`
echo $entry
}
DSGetLogSummary()
{
logsum=`cpdctl dsjob logsum --project-id $proj_id --job-id $job_id --run-id $run_id`
echo $logsum
}
The following function calls get the log
information.$ logids=`DSGetLogEventIds`
$ lastid=`DSGetNewestLogId`
$ logstr=`DSGetLogEntry $lastid`
$ logsum=`DSGetLogSummary`
- DSGetParamInfo
- To get the current value of a parameter, get the project id, job id, and run id. Write a wrapper
function to retrieve the job run metadata.
- REST API:
-
DSGetParamInfo()
{
# Get the job run metadata
jobrun=`curl -k -s --request GET "${CPD_URL}/v2/jobs/${job_id}/runs/${run_id}?job_id=${job_id}&run_id=${run_id}&project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
# Extract the parameter value
params=`echo $jobrun |jq -r '.entity.job_run.configuration.job_parameters[]' | grep "${1}="`
echo $params | awk -F '=' '{print $2}'
}
- dsjob CLI:
DSGetParamInfo()
{
# Get the job run metadata
jobrun=`cpdctl dsjob get-jobrun --project-id $proj_id --id $job_id --run-id $run_id --output json`
# Extract the parameter value
params=`echo $jobrun |jq -r '.job_run.configuration.job_parameters[]' | grep "${1}="`
echo $params | awk -F '=' '{print $2}'
}
Call the function to get a parameter
value.$ value=`DSGetParamInfo "param_name"`
- DSGetProjectInfo
- To get the name of the current project and a comma-separated list of the project's jobs, you
need the project id. These wrapper functions return the name and list.
- REST API:
-
#!/bin/sh
DSGetProjectInfo()
{
if [ "$1" = "DSJ.PROJECTNAME" ]; then
# Get the project metadata
proj=`curl -k -s --request GET "${CPD_URL}/v2/projects/${proj_id}?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
# Parse and extract the name
echo $proj |jq -r '.entity.name'
elif [ "$1" = "DSJ.JOBLIST" ]; then
# Get the flow list metadata
flows=`curl -k -s --request GET "${CPD_URL}/data_intg/v3/data_intg_flows?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
# Parse and extract the flow names
echo $flows|jq -r '.data_flows[].metadata.name'
fi
}
- dsjob CLI:
#!/bin/sh
DSGetProjectInfo()
{
if [ "$1" = "DSJ.PROJECTNAME" ]; then
# List projects and pick up needed one based on project id
proj=`cpdctl dsjob list-projects --with-id |grep $proj_id`
# Parse the output and extract the name
echo $proj|awk '{print $1}'
elif [ "$1" = "DSJ.JOBLIST" ]; then
# Get the flow list and extract the flow names
flows=`cpdctl dsjob list-flows --project-id ${proj_id} | tac | sed '1,4d;$d' | tac`
echo $flows
fi
}
Call the function to get the project name or flow
list.$ projectname=`DSGetProjectInfo "DSJ.PROJECTNAME"`
$ flowlist=`DSGetProjectInfo "DSJ.JOBLIST"`
- DSGetVersionInfo
- The following function uses the REST API to get the product version.
-
DSGetVersionInfo()
{
if [ "$1" = "DSJ.VER.DS" ]; then
prod="DataStage"
elif [ "$1" = "DSJ.VER.DATAREP" ]; then
prod="data-replication"
elif [ "$1" = "DSJ.VER.PIPELINE" ]; then
prod="Watson Studio Pipelines"
elif [ "$1" = "DSJ.VER.CORE" ]; then
prod="Common Core Services"
elif [ "$1" = "DSJ.VER.CONTROL" ]; then
prod="Control plane"
fi
versions=`curl -k -s --request GET "${CPD_URL}/zen-data/v1/extensions?extension_point_id=zen_services_info" --header "Authorization: Bearer ${bearer_token}"`
echo $versions | jq -r ".data[]|select(.display_name==\"$prod\")|.details.version"
}
Call the function to get the product version.$ DSGetVersionInfo "DSJ.VER.DS"
4.6.0
- DSJobNameFromJobId
- To get a job name, you need the project id and job id. The following functions retrieve a job
name.
- REST API:
-
DSJobNameFromJobId()
{
job=`curl --request GET "${CPD_URL}/v2/jobs/${job_id}?project_id=${proj_id}&job_id=${job_id}" --header "Authorization: Bearer ${bearer_token}" -k -s`
echo $job | jq -r '.metadata.name'
}
- dsjob CLI:
DSJobNameFromJobId()
{
job=`cpdctl dsjob get-job --project-id $proj_id --id $job_id --output json --with-metadata | sed '$d'`
echo $job | jq -r '.metadata.name'
}
Call the function to get the job
name.jobname=`DSJobNameFromJobId``
- DSListEnvVars
- To list environment variables, you need the project id.
- REST API:
-
#!/bin/sh
# param1: environment name
DSListEnvVars()
{
envs=`curl -k -s --request GET "${CPD_URL}/v2/environments?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
vars=`echo $envs|jq -r ".resources[] | select(.metadata.name==\"$1\") | .entity.environment.environment_variables"`
echo $vars
}
- dsjob CLI:
#!/bin/sh
# param1: environment name
DSListEnvVars()
{
output=`cpdctl dsjob list-env-vars --project-id ${proj_id} -n $1`
vars=`echo $output |sed '1d' |tac | sed '1,4d'`
echo $vars
}
Call the function to get the
list.$ DSListEnvVars "default_datastage_px"
- DSMakeJobReport
- To make a job report, define DSGetJobInfo, DSGetStageInfo and DSGetLinkInfo. The following
function shows the status of a job run.
-
DSMakeJobReport()
{
# job basic info
jobname=`DSGetJobInfo "DSJ.JOBNAME"`
jobelapsed=`DSGetJobInfo "DSJ.JOBELAPSED"`
jobstatus=`DSGetJobInfo "DSJ.JOBSTATUS"`
userstatus=`DSGetJobInfo "DSJ.USERSTATUS"`
echo "STATUS REPORT FOR JOB: ${jobname}"
echo "Job elapsed time=${jobelapsed}"
echo "Job status=${jobstatus}"
if [ "$userstatus" != "" ]; then
echo "User status=${userstatus}"
fi
if [ "$1" != "1" ]; then
exit 0
fi
# stage and link detail
stagelist=`DSGetJobInfo "DSJ.STAGELIST"`
stagearray=($stagelist)
for stage in ${stagearray[@]}
do
rows=`DSGetStageInfo "DSJ.STAGEINROWNUM" "${stage}"`
echo " Stage: ${stage}, ${rows} rows input"
links=`DSGetStageInfo "DSJ.LINKLIST" "${stage}"`
linkarray=($links)
for link in ${linkarray[@]}
do
rows=`DSGetLinkInfo "DSJ.LINKROWCOUNT" ${link}`
echo " Link: ${link}, ${rows} rows"
done
done
}
- Call the function to get the list.
DSMakeJobReport "1"
- DSRunJob
- REST API:
#!/bin/sh
DSRunJob()
{
if [ "$1" = "" ]; then
envs='{}'
else
envs="{\"job_run\": {\"configuration\": {\"env_variables\": [\"$1\"]}}}"
fi
jobrun=`curl -k -s --request POST "${CPD_URL}/v2/jobs/${job_id}/runs?job_id=${job_id}&project_id=${proj_id}" --header 'Content-Type: application/json' --header "Authorization: Bearer ${bearer_token}" -d "$envs"`
run_id=`echo $jobrun | jq -r '.metadata.asset_id'`
echo $run_id
}
- dsjob CLI:
#!/bin/sh
DSRunJob()
{
if [ "$1" = "" ]; then
envs=""
else
envs="--env $1"
fi
output=`cpdctl dsjob run --project-id ${proj_id} --job-id ${job_id} $envs`
run_id=`echo $output | grep 'ID' | awk '{print $4}'`
echo $run_id
}
The following syntax runs a specified
job.
# get the job_id according to job name
$ joblist=`curl --request GET "${CPD_URL}/v2/jobs?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}" -k -s`
$ job_id=`echo $joblist|jq -r '.results[].metadata|select(.name=="dstestjob"?) |.asset_id'`
# run job
$ run_id=`DSRunJob "key1=value1"`
- DSSetEnvVar
- REST API:
#!/bin/sh
# param1: environment name
# param2: variable name
# param3: variable value
DSSetEnvVar()
{
envs=`curl -k -s --request GET "${CPD_URL}/v2/environments?project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
env_id=`echo $envs|jq -r ".resources[] | select(.metadata.name==\"$1\") | .metadata.asset_id"`
existingvars=`echo $envs|jq -r ".resources[] | select(.metadata.name==\"$env_name\") | .entity.environment.environment_variables"`
updatevars=`echo $existingvars |jq ".$2 = \"$3\""`
data="{\"/entity/environment/environment_variables\": ${updatevars}}"
curl -k -s --request PATCH "${CPD_URL}/v2/environments/${env_id}?project_id=${proj_id}&environment_guid=${proj_id}" --header 'Content-Type: application/json' --header "Authorization: Bearer ${bearer_token}" --data-raw "${data}"
}
- dsjob CLI:
#!/bin/sh
# param1: environment name
# param2: variable name
# param3: variable value
DSSetEnvVar()
{
cpdctl dsjob update-env-vars --project-id ${proj_id} -n $1 --env $2=$3
}
The following syntax sets an environment variable to the specified value.
$ DSSetEnvVar "default_datastage_px" "uservar" "newval"
- DSSetIdForJob
- Use the REST API to set a user-defined name for a job.
#!/bin/sh
#param1: job id
#param2: new job name
DSSetIdForJob()
{
data="[{\"op\": \"replace\", \"path\": \"/metadata/name\", \"value\": $2}]"
curl -k -s --request PATCH "${CPD_URL}/v2/jobs/$1?job_id=$1&project_id=${proj_id}" --header 'Content-Type: application/json' --header "Authorization: Bearer ${bearer_token}" --data-raw "${data}"
}
Call the function to set the
name.$ DSSetIdForJob $job_id "new_job_name"
- DSSetParam
- Use the REST API to specify job parameter values.
#!/bin/sh
#param1: job id
#param2: parameter name
#param3: parameter value
DSSetParam()
{
vars=(`curl -k -s --request GET "${CPD_URL}/v2/jobs/$1?job_id=$1&project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}" | jq '.entity.job.configuration.job_parameters[]'`)
for i in "${!vars[@]}";
do
if [[ ${vars[$i]} == \"$2=* ]] ; then
newvars[$i]=\"$2=$3\"
else
newvars[$i]=${vars[$i]}
fi
done
varstr=$(printf ",%s" "${newvars[@]}")
varstr="[${varstr:1}]"
data="[{\"op\": \"replace\", \"path\": \"/entity/job/configuration/job_parameters\", \"value\": ${varstr}}]"
curl -k -s --request PATCH "${CPD_URL}/v2/jobs/$1?job_id=$1&project_id=${proj_id}" --header 'Content-Type: application/json' --header "Authorization: Bearer ${bearer_token}" --data-raw "${data}"
}
Call the function to set the
value.$ DSSetParam $job_id "param" "newvalue"
- DSStopJob
- REST API:
#!/bin/sh
DSStopJob()
{
output=`curl -k -s --request POST "${CPD_URL}/v2/jobs/${job_id}/runs/${run_id}/cancel?job_id=${job_id}&project_id=${proj_id}&run_id=${run_id}" --header 'Content-Type: application/json' --header "Authorization: Bearer ${bearer_token}" -d '{}'`
echo $output
}
- dsjob CLI:
#!/bin/sh
DSStopJob()
{
output=`cpdctl dsjob stop --project-id $proj_id --job-id $job_id --run-id $run_id`
echo $output
}
The following syntax stops a job. The job stopped is the one specified in the
job_id parameter. $ DSStopJob
- DSTranslateCode
- Define a translation function to convert a job control status or error code into a text
explanation.
-
#!/bin/sh
DSTranslateCode()
{
case $1 in
0)
echo "Running"
;;
1)
echo "Finished OK"
;;
2)
echo "Finished with warnings"
;;
3)
echo "Aborted"
;;
4)
echo "Queued"
;;
11)
echo "Validated OK"
;;
12)
echo "Validated with warnings"
;;
13)
echo "Failed validation"
;;
21)
echo "Has been reset"
;;
96)
echo "Crashed"
;;
97)
echo "Stopped by operator"
;;
98)
echo "Not compiled"
;;
99)
echo "Not running"
;;
-1)
echo "Invalid JobHandle"
;;
-2)
echo "Job is not in the right state (compiled and not running)"
;;
-3)
echo "ParamName does not reference a known parameter of the job"
;;
-4)
echo "ParamValue/Limitvalue is not appropriate"
;;
-5)
echo "Token in LimitType/RunMode/InfoType not recognized"
;;
-6)
echo "Job was not run from within the current job"
;;
-7)
echo "StageName does not refer to a known stage in the job"
;;
-8)
echo "StageName was DSJ.ME and the caller is not running within a stage"
;;
-9)
echo "LinkName does not refer to a known link of the stage"
;;
-10)
echo "JobHandle refers to a job that is locked by another user"
;;
-11)
echo "JobHandle refers to a job that has now been deleted"
;;
-12)
echo "JobName badly formed or job does not exist"
;;
-13)
echo "A TimeStamp parameter was badly formed"
;;
-14)
echo "Timed out while waiting for an event"
;;
-15)
echo "Decryption of encrypted value failed"
;;
-16)
echo "Cannot get values, default values or design default values for any job except the current job"
;;
-17)
echo "Cannot find template file"
;;
-18)
echo "Error encountered processing template file"
;;
-19)
echo "Parameter name missing - field does not look like 'name:value'"
;;
-20)
echo "File path name not given"
;;
-21)
echo "Error when executing external command"
;;
-99)
echo "General repository interface 'other error'"
;;
*)
echo "Unknown status $1"
;;
esac
}
- DSWaitForJob
- REST
API:
$ jobrun=`curl -k -s --request GET "${CPD_URL}/v2/jobs/${job_id}/runs/${run_id}?job_id=${job_id}&run_id=${run_id}&project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
$ status=`echo $jobrun | jq -r '.entity.job_run.state'`
$ echo $status
- dsjob
CLI:
$ jobrun=`cpdctl dsjob get-jobrun --project-id ${proj_id} -N ${job_id} --run-id ${run_id} --output json`
$ status=`echo $jobrun | jq -r '.job_run.state'`
$ echo $status
Wrapper function:#!/bin/sh
#param1: job id
#param2: run id
#param3: timeout
DSWaitForJob()
{
OLD_IFS="$IFS"
IFS=","
jobarr=($1)
runarr=($2)
IFS="$OLD_IFS"
len=${#jobarr[@]}
start=$(date +%s)
while true
do
now=$(date +%s)
time=$(( $now - $start ))
if [ $time -ge $3 ]; then
return 1
fi
result=0
for ((i=0;i<len;i++)) do
if [ ! -z "${statarr[i]}" ] && [ ${statarr[i]} = "Completed" ]; then
result=$(($result+1))
continue
fi
job_id=${jobarr[i]}
run_id=${runarr[i]}
jobrun=`cpdctl dsjob get-jobrun --project-id ${proj_id} -N ${job_id} --run-id ${run_id} --output json`
status=`echo $jobrun | jq -r '.job_run.state'`
statarr[i]=$status
if [ ${statarr[i]} = "Completed" ]; then
result=$(($result+1))
elif [ ${statarr[i]} = "Completed" ]; then
return 1
fi
done;
if [ $result -eq $len ]; then
return 0
fi
done
}
The following syntax waits for a specified
job.$ DSWaitForJob "${jobid1},${jobid2}" "${runid1},${runid2}" "600"
- DSJobLinkInfo
- Use the REST API to get link information from job run metadata.
DSJobLinkInfo()
{
# Get the job run metadata
jobrun=`curl -k -s --request GET "${CPD_URL}/v2/jobs/${job_id}/runs/${run_id}?job_id=${job_id}&run_id=${run_id}&project_id=${proj_id}" --header "Authorization: Bearer ${bearer_token}"`
# Get link information
echo $jobrun | jq -r ".entity.job_run.configuration.metrics.link_metrics[]|(\"Link name:\" + .link_name + \", source:\" + .source + \", dest:\" + .dest + \", state:\" + .state + \", rows read:\" + (.rows_read|tostring) + \", rows written:\" + (.rows_written|tostring))"
}
Call the function to get the link info.$ rows=`DSJobLinkInfo`