ndr_avro-lambda/lambda_function.rb at main · HealthDataInsight/ndr_avro-lambda · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
require 'rubygems'
require 'bundler/setup'

# require 'aws-sdk-lambda'
require 'json'
require 'logger'
require 'ndr_avro'
require 'open-uri'
require_relative 'safe_dir'

# $client = Aws::Lambda::Client.new()
# $client.get_account_settings()

# require 'aws-xray-sdk/lambda'

# Configure SafePath
SafePath.configure!(File.join('.', 'filesystem_paths.yml'))

# Configure NdrImport::StandardMappings
mappings_path = SafePath.new('root').join('standard_mappings.yml')
NdrImport::StandardMappings.mappings = YAML.load_file(mappings_path)

# Main class containing the entry point class method LambdaFunction.process
class LambdaFunction
  VERSION = '0.1.0'.freeze

  def self.process(event:, context:)
    logger = Logger.new($stdout)
    # logger.info('## ENVIRONMENT VARIABLES')
    # vars = Hash.new
    # ENV.each do |variable|
    #   vars[variable[0]] = variable[1]
    # end
    # logger.info(vars.to_json)
    logger.info('## EVENT')
    logger.info(event.to_json)
    # logger.info('## CONTEXT')
    # logger.info(context)

    # Set object details
    t0 = Time.current

    s3_event_hash = event['Records'].first['s3']
    input_bucket = s3_event_hash['bucket']['name']
    # input_bucket = s3_event_hash['bucket']['arn']
    object_key = s3_event_hash['object']['key']

    output_bucket = ENV['OUTPUT_BUCKET_NAME']
    mappings = URI.parse(ENV['MAPPING_URL']).read
    # logger.info('## MAPPING_URL')
    # logger.info(ENV['MAPPING_URL'])
    # logger.info('## MAPPINGS')
    # logger.info(mappings)

    SafeDir.mktmpdir do |safe_dir|
      s3_wrapper = NdrAvro::S3Wrapper.new(safe_dir: safe_dir)

      # Create a temporary copy of the mappings
      table_mappings = s3_wrapper.materialise_mappings(mappings)

      # Create a temporary copy of the S3 file
      safe_input_path = s3_wrapper.get_object(input_bucket, object_key)

      t1 = Time.current

      # Generate the avro file(s)
      generator = NdrAvro::Generator.new(safe_input_path, table_mappings, safe_dir)
      generator.process

      t2 = Time.current

      results = []

      # Put the output files in the output S3 bucket
      generator.output_files.each do |output_file_hash|
        object_hash = s3_wrapper.put_object(output_bucket, output_file_hash[:path])
        results << object_hash.merge(total_rows: output_file_hash[:total_rows])

        object_hash = s3_wrapper.put_object(output_bucket, output_file_hash[:schema])
        results << object_hash
      end

      t3 = Time.current

      logger.info(results)

      # return {
      #   results: results,
      #   timings: {
      #     s3_get: t1 - t0,
      #     generator: t2 - t1,
      #     s3_put: t3 - t2,
      #     total: t3 - t0
      #   },
      #   versions: {
      #     lambda_function: LambdaFunction::VERSION,
      #     ndr_import: NdrImport::VERSION,
      #     ndr_avro: NdrAvro::VERSION,
      #     ruby: RUBY_VERSION
      #   }
      # }
    rescue StandardError => e
      logger.info('## ERROR')
      logger.info(e.class)
      logger.info(e.message)
      logger.info(e.backtrace)

    #   return {
    #     error: {
    #       class: e.class,
    #       message: e.message,
    #       backtrace: e.backtrace
    #     }
    #   }
    end
    # $client.get_account_settings().account_usage.to_h
  end
end