Merge pull request #142 from fagai/support_description

Support description of columns and tables

Author: hiroyuki-sato

README.md

@@ -60,6 +60,7 @@ OAuth flow for installed applications.
 |  gcs_bucket                          | string      | optional   | nil                      | See [GCS Bucket](#gcs-bucket) |
 |  auto_create_gcs_bucket              | boolean     | optional   | false                    | See [GCS Bucket](#gcs-bucket) |
 |  progress_log_interval               | float       | optional   | nil (Disabled)           | Progress log interval. The progress log is disabled by nil (default). NOTE: This option may be removed in a future because a filter plugin can achieve the same goal |
+|  description                         | string      | optional   | nil                      | description of table |
 
 Client or request options
 
@@ -329,6 +330,7 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
     - json:      `STRING`,  `RECORD` (default: `STRING`)
   - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
   - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
+  - **description**: description (string, default is `None`).
   - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`)
   - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`).
 - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N")

lib/embulk/output/bigquery.rb

@@ -63,6 +63,8 @@ def self.configure(config, schema, task_count)
           'payload_column'                 => config.param('payload_column',                 :string,  :default => nil),
           'payload_column_index'           => config.param('payload_column_index',           :integer, :default => nil),
 
+          'description'                    => config.param('description',                    :string,  :default => nil),
+
           'open_timeout_sec'               => config.param('open_timeout_sec',               :integer, :default => nil),
           'timeout_sec'                    => config.param('timeout_sec',                    :integer, :default => nil), # google-api-ruby-client < v0.11.0
           'send_timeout_sec'               => config.param('send_timeout_sec',               :integer, :default => nil), # google-api-ruby-client >= v0.11.0

lib/embulk/output/bigquery/bigquery_client.rb

@@ -420,6 +420,7 @@ def create_table_if_not_exists(table, dataset: nil, options: nil)
               table_reference: {
                 table_id: table,
               },
+              description: @task['description'],
               schema: {
                 fields: fields,
               }

lib/embulk/output/bigquery/helper.rb

@@ -46,10 +46,11 @@ def self.fields_from_embulk_schema(task, schema)
             embulk_type   = column[:type]
             column_option = column_options_map[column_name] || {}
             {}.tap do |field|
-              field[:name]   = column_name
-              field[:type]   = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase
-              field[:mode]   = column_option['mode'] if column_option['mode']
-              field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields']
+              field[:name]        = column_name
+              field[:type]        = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase
+              field[:mode]        = column_option['mode'] if column_option['mode']
+              field[:fields]      = deep_symbolize_keys(column_option['fields']) if column_option['fields']
+              field[:description] = column_option['description'] if column_option['description']
             end
           end
         end

test/test_helper.rb

@@ -68,7 +68,7 @@ def test_fields_from_embulk_schema_with_column_options
           ])
           task = {
             'column_options' => [
-              {'name' => 'boolean',   'type' => 'STRING', 'mode' => 'REQUIRED'},
+              {'name' => 'boolean',   'type' => 'STRING', 'mode' => 'REQUIRED', 'description' => 'hoge'},
               {'name' => 'long',      'type' => 'STRING'},
               {'name' => 'double',    'type' => 'STRING'},
               {'name' => 'string',    'type' => 'INTEGER'},
@@ -81,7 +81,7 @@ def test_fields_from_embulk_schema_with_column_options
             ],
           }
           expected = [
-            {name: 'boolean',   type: 'STRING', mode: 'REQUIRED'},
+            {name: 'boolean', type: 'STRING', mode: 'REQUIRED', description: 'hoge'},
             {name: 'long',      type: 'STRING'},
             {name: 'double',    type: 'STRING'},
             {name: 'string',    type: 'INTEGER'},