ny-tlc-report.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: cm-job-arguments
data:
  job-args.txt: |
    s3a://my-bucket/yellow_tripdata_2021-07.csv
---
apiVersion: spark.stackable.tech/v1alpha1
kind: SparkApplication
metadata:
  name: spark-ny-cm
spec:
  sparkImage:
    productVersion: 3.5.5
  mode: cluster
  mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.5.jar
  mainClass: tech.stackable.demo.spark.NYTLCReport
  volumes:
    - name: cm-job-arguments
      configMap:
        name: cm-job-arguments
  args:
    - "--input /arguments/job-args.txt"
  s3connection:
    inline:
      host: test-minio
      port: 9000
      accessStyle: Path
  sparkConf:
    spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
  driver:
    config:
      volumeMounts:
        - name: cm-job-arguments
          mountPath: /arguments
  executor:
    replicas: 3
    config:
      volumeMounts:
        - name: cm-job-arguments
          mountPath: /arguments