CODEWITHSUNDEEP
scalaaws-glue
Max08
Max08

Reputation: 1025

Getting workflow runtime properties for AWS Glue workflow in Scala

I am working on an AWS Glue job. I am using scala to write the code. I need to get the workflow runtime properties. I can do this very easily in python. However i could not find any sample code or documentation to do this in scala.

Equivalent code in python is as follows.

I will be very grateful if someone can help me with the scala equivalent.


import sys
import boto3
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from awsglue.context import GlueContext
from pyspark.context import SparkContext

glue_client = boto3.client("glue")
args = getResolvedOptions(sys.argv, ['JOB_NAME','WORKFLOW_NAME', 'WORKFLOW_RUN_ID'])
workflow_name = args['WORKFLOW_NAME']
workflow_run_id = args['WORKFLOW_RUN_ID']
workflow_params = glue_client.get_workflow_run_properties(Name=workflow_name,
                                        RunId=workflow_run_id)["RunProperties"]

target_database = workflow_params['target_database']
target_s3_location = workflow_params['target_s3_location']

Upvotes: 1

Views: 736

Answers (1)

Rasputin
Rasputin

Reputation: 11

This worked for me.

import com.amazonaws.regions.Regions

import com.amazonaws.services.glue.{AWSGlue, AWSGlueClient}
import com.amazonaws.services.glue.model.GetWorkflowRunPropertiesRequest
import com.amazonaws.services.glue.model.GetWorkflowRunPropertiesResult


import com.amazonaws.services.glue.GlueContext
import com.amazonaws.services.glue.util.GlueArgParser
import com.amazonaws.services.glue.util.Job  

import org.apache.spark.sql.SparkSession
import org.apache.spark.SparkContext
import com.amazonaws.services.glue.GlueContext  

object ReadProps {


  def main(sysArgs: Array[String]) {
      val args = GlueArgParser.getResolvedOptions(sysArgs, Seq("JOB_NAME","WORKFLOW_NAME", "WORKFLOW_RUN_ID").toArray)
      val workflowName= args("WORKFLOW_NAME")
      val workflowId = args("WORKFLOW_RUN_ID")
      
      val sc: SparkContext = new SparkContext()
      val glueContext: GlueContext = new GlueContext(sc)
      val sparkSession: SparkSession = glueContext.getSparkSession
      
      val region = Regions.fromName("your-region-name")
      
      val glue = AWSGlueClient.builder().withRegion(region).build()
      
      val req = new GetWorkflowRunPropertiesRequest()
      req.setName(workflowName)
      req.setRunId(workflowId)
      
      val result = glue.getWorkflowRunProperties(req)
      val resultMap = result.getRunProperties()
      println(resultMap.get("propertykey"))
      

      
  }
  
}

Upvotes: 1

Related Questions