application.conf

deepdive {

  sampler.sampler_args: "-l 300 -s 1 -i 500 --alpha 0.1 -c 0"

  schema.variables {
    locations.is_correct: Boolean
  }

  pipeline.run: all 

  pipeline.pipelines {
    preprocess = [ extract_preprocess ]
    entity_features_only = [ extract_pairs, one_of_n_features, consecutive_in_proximity,
      city, city_with_hundreds_of_thousands_of_inhabitants, city_with_millions_of_inhabitants ]
    all_features = [ extract_pairs, extract_context_features, one_of_n_features, consecutive_in_proximity,
      city, city_with_hundreds_of_thousands_of_inhabitants, city_with_millions_of_inhabitants, context_features ]
    all = [ extract_preprocess, extract_pairs, extract_context_features, one_of_n_features, consecutive_in_proximity,
      city, city_with_hundreds_of_thousands_of_inhabitants, city_with_millions_of_inhabitants, context_features ]
  }

  db.default {
    driver   : "org.postgresql.Driver"
    url      : "jdbc:postgresql://"${PGHOST}":"${PGPORT}"/"${DBNAME}
    user     : ${PGUSER}
    password : ${PGPASSWORD}
    dbname   : ${DBNAME}
    host     : ${PGHOST}
    port     : ${PGPORT}
    gphost   : ${GPHOST}
    gpport   : ${GPPORT}
    gppath   : ${GPPATH}
    # start gpfdist server on the machine running the application with
    # `rungpcommand 'gpfdist -d /lfs/raiders4/0/rionda/greenplum_gpfdist-memex -p 9999'`
  }

  calibration.holdout_fraction: 0.075

  extraction.extractors {

   extract_preprocess: {
       style: json_extractor
       before: psql -h ${PGHOST} -p ${PGPORT} -d ${DBNAME} -f ${APP_HOME}/schemas/sentences.sql
       input: """
               SELECT id,
                 body
               FROM articles
               WHERE NOT BODY IS NULL
               ORDER BY id ASC
              """
       output_relation: sentences
       udf: ${DEEPDIVE_HOME}/examples/nlp_extractor/run.sh -k id -v body -l 100 -t 16 -a "tokenize,ssplit,pos"
   }

    extract_pairs: {
        style: tsv_extractor
        before: psql -h ${PGHOST} -p ${PGPORT} -d ${DBNAME} -f ${APP_HOME}/schemas/locations.sql
        input: """
                SELECT
                        sentence_id,
                        array_to_string(words, ' '),
                        array_to_string(pos_tags, ' ')
                FROM sentences;
                """
        output_relation: locations
        udf: ${APP_HOME}/udf/extract_pairs.py
    }

    extract_context_features: {
        style: tsv_extractor
        before: psql -h ${PGHOST} -p ${PGPORT} -d ${DBNAME} -f ${APP_HOME}/schemas/context_features.sql
        input: """
                SELECT
                        l.sentence_id, mention_num, w_from, w_to, 
                        array_to_string(words, ' '),
                        array_to_string(pos_tags, ' ')
                FROM sentences s, (SELECT DISTINCT sentence_id, mention_num, w_from, w_to FROM locations) l
                WHERE s.sentence_id = l.sentence_id;
                """
        output_relation: context_features
        udf: ${APP_HOME}/udf/extract_context_features.py
    }

    supervise: {
        style: tsv_extractor
        input: """
               SELECT
               
               FROM sentences
               """
        output_relation: locations
        udf: ${APP_HOME}/udf/supervise.py
    }
  }
  
  inference.factors {

    one_of_n_features {
      input_query = """
        SELECT l1.id as "linking1.id", l1.is_correct as "linking1.is_correct",
               l2.id as "linking2.id", l2.is_correct as "linking2.is_correct"
        FROM locations l1, locations l2
        WHERE l1.sentence_id = l2.sentence_id 
        AND l1.mention_num = l2.mention_num
        AND NOT l1.mention_id = l2.mention_id;
         """
      function: "And(linking1.is_correct, linking2.is_correct)"
      weight: -10
    }

    # prefer if subsequently mentioned cities are within 1000km distance
    consecutive_in_proximity {
      input_query: """
        SELECT l1.id as "linking1.id", l1.is_correct as "linking1.is_correct",
               l2.id as "linking2.id", l2.is_correct as "linking2.is_correct"
        FROM locations l1, locations l2,
             wikidata_coordinate_locations c1, wikidata_coordinate_locations c2
        WHERE l1.loc_id = c1.item_id
        AND l2.loc_id = c2.item_id
        AND l1.sentence_id = l2.sentence_id
        AND l2.mention_num = l1.mention_num + 1
        AND earth_distance(ll_to_earth(c1.latitude,c1.longitude), ll_to_earth(c2.latitude,c2.longitude)) < 1000;
        """
      function: "And(linking1.is_correct, linking2.is_correct)"
      weight: "3"
    }

    # prefer larger cities
    city {
      input_query: """
        SELECT l.id as "linking.id", l.is_correct as "linking.is_correct"
        FROM locations l, wikidata_instanceof i
        WHERE l.loc_id = i.item_id
        AND i.clazz_id = 515;
        """
      function: "IsTrue(linking.is_correct)"
      weight: 1
    }
    
    city_with_hundreds_of_thousands_of_inhabitants {
      input_query: """
        SELECT l.id as "linking.id", l.is_correct as "linking.is_correct"
        FROM locations l, wikidata_instanceof i
        WHERE l.loc_id = i.item_id
        AND i.clazz_id = 1549591;
        """
      function: "IsTrue(linking.is_correct)"
      weight: 1
    }
               
    city_with_millions_of_inhabitants {
      input_query: """
        SELECT l.id as "linking.id", l.is_correct as "linking.is_correct"
        FROM locations l, wikidata_instanceof i
        WHERE l.loc_id = i.item_id
        AND i.clazz_id = 1637706;
        """
      function: "IsTrue(linking.is_correct)"
      weight: 1
    } 

    boost_countries {
      input_query: """
        SELECT l.id as "linking.id", l.is_correct as "linking.is_correct"
        FROM locations l, wikidata_instanceof i
        WHERE l.loc_id = i.item_id
        AND i.clazz_id = 6256;
        """
      function: "IsTrue(linking.is_correct)"
      weight: 3
    }

    context_features {
      input_query = """
        SELECT l.id as "locations.id", l.is_correct as "locations.is_correct", unnest(f.features) as "locations.feature"
        FROM locations l, context_features f
        WHERE l.sentence_id = f.sentence_id
        AND l.mention_num = f.mention_num;
        """
      function: "IsTrue(locations.is_correct)"
      weight: "?(locations.feature)"
    }

  }
}