Add search_tags_count.

Author: keillera

function02-template.yaml

@@ -311,6 +311,23 @@ Resources:
       Timeout: 15
       TracingConfig:
         Mode: "Active"
+  SearchTagsCount:
+    Type: "AWS::Lambda::Function"
+    Properties:
+      Code: ./deploy/articles_tip_ranking.zip
+      Environment:
+        Variables:
+          ELASTIC_SEARCH_ENDPOINT: !Ref ElasticSearchEndpoint
+      Handler: handler.lambda_handler
+      MemorySize: 3008
+      Role:
+        Fn::ImportValue:
+          Fn::Sub: "${AlisAppId}-LambdaRole"
+      Runtime: python3.6
+      Timeout: 300
+      TracingConfig:
+        Mode: "Active"
+
 
 Outputs:
   ArticleSupportersIndex:
@@ -373,3 +390,7 @@ Outputs:
     Value: !GetAtt TopicsCryptoRankingIndex.Arn
     Export:
       Name: !Sub "${AlisAppId}-TopicsCryptoRankingIndex"
+  SearchTagsCount:
+    Value: !GetAtt SearchTagsCount.Arn
+    Export:
+      Name: !Sub "${AlisAppId}-SearchTagsCount"

src/common/es_util.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import settings
 import os
+import time
 
 
 class ESUtil:
@@ -38,6 +39,35 @@ def search_tag(elasticsearch, word, limit, page):
 
         return tags
 
+    @staticmethod
+    def search_tags_count(elasticsearch, size=500, term=604800):
+        from_time = round(time.time()) - term
+        body = {
+          'size': 0,
+          'query': {
+            'bool': {
+              'must': [
+                {'range': {'published_at': {'gte': from_time}}}
+              ]
+            }
+          },
+          'aggs': {
+            'tags_count': {
+              'terms': {
+                'field': 'tags.keyword',
+                'order': {'_count': 'desc'},
+                'size': size
+              }
+            }
+          }
+        }
+
+        response = elasticsearch.search(
+            index='articles',
+            body=body
+        )
+        return response['aggregations']['tags_count']['buckets']
+
     @staticmethod
     def search_article(elasticsearch, limit, page, word=None, tag=None):
         body = {

src/common/settings.py

@@ -150,6 +150,15 @@
         },
         'maxItems': 5
     },
+    'tags_count': {
+        'type': 'array',
+        'items': {
+            'type': 'string',
+            'minLength': 1,
+            'maxLength': 25
+        },
+        'maxItems': 150
+    },
     'tip_value': {
         'type': 'number',
         'minimum': 1,

src/handlers/search/tags_count/handler.py

@@ -0,0 +1,24 @@
+import os
+from search_tags_count import SearchTagsCount
+from elasticsearch import Elasticsearch, RequestsHttpConnection
+from requests_aws4auth import AWS4Auth
+
+awsauth = AWS4Auth(
+    os.environ['AWS_ACCESS_KEY_ID'],
+    os.environ['AWS_SECRET_ACCESS_KEY'],
+    os.environ['AWS_REGION'],
+    'es',
+    session_token=os.environ['AWS_SESSION_TOKEN']
+)
+elasticsearch = Elasticsearch(
+    hosts=[{'host': os.environ['ELASTIC_SEARCH_ENDPOINT'], 'port': 443}],
+    http_auth=awsauth,
+    use_ssl=True,
+    verify_certs=True,
+    connection_class=RequestsHttpConnection
+)
+
+
+def lambda_handler(event, context):
+    search_tags_count = SearchTagsCount(event, context, elasticsearch=elasticsearch)
+    return search_tags_count.main()

src/handlers/search/tags_count/search_tags_count.py

@@ -0,0 +1,42 @@
+import json
+
+from jsonschema import validate
+
+import settings
+from decimal_encoder import DecimalEncoder
+from es_util import ESUtil
+from lambda_base import LambdaBase
+
+
+class SearchTagsCount(LambdaBase):
+    def get_schema(self):
+        return {
+            'type': 'object',
+            'properties': {
+                'tags': settings.parameters['tags_count']
+            },
+            'required': ['tags']
+        }
+
+    def validate_params(self):
+        validate(self.params, self.get_schema())
+
+    def exec_main_proc(self):
+        # 直近１週間分のタグを集計
+        search_size = len(self.params['tags']) * settings.parameters['tags']['maxItems']
+        from_time = 86400 * 7
+        search_result = ESUtil.search_tags_count(self.elasticsearch, search_size, from_time)
+
+        # 集計結果より指定タグの件数を取得
+        temp = []
+        for tag in self.params['tags']:
+            tag_count = [d['doc_count'] for d in search_result if d['key'] == tag]
+            temp.append({
+                'tag': tag,
+                'count': tag_count[0] if len(tag_count) > 0 else 0
+            })
+        result = sorted(temp, key=lambda x: x['count'], reverse=True)
+        return {
+            'statusCode': 200,
+            'body': json.dumps(result, cls=DecimalEncoder)
+        }

swagger/swagger.yaml

@@ -256,6 +256,13 @@ definitions:
         type: integer
       created_at:
         type: integer
+  TagCount:
+    type: object
+    properties:
+      tag:
+        type: string
+      count:
+        type: integer
   Fraud:
     type: object
     properties:
@@ -498,6 +505,38 @@ paths:
         passthroughBehavior: when_no_templates
         httpMethod: POST
         type: aws_proxy
+  /search/tags_count:
+    get:
+      description: "タグ件数検索"
+      parameters:
+      - name: "tags"
+        in: "query"
+        description: "検索タグ"
+        required: true
+        type: array
+        items:
+          type: string
+      responses:
+        "200":
+          description: "タグ件数一覧"
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/TagCount'
+      x-amazon-apigateway-integration:
+        responses:
+          default:
+            statusCode: "200"
+        uri:
+          Fn::Join:
+            - ''
+            - - Fn::Sub: "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/"
+              - Fn::ImportValue:
+                  Fn::Sub: "${AlisAppId}-SearchTagsCount"
+              - "/invocations"
+        passthroughBehavior: when_no_templates
+        httpMethod: POST
+        type: aws_proxy
   /articles/recent:
     get:
       description: "最新記事一覧情報を取得"

tests/handlers/search/tags_count/test_search_tags_count.py

@@ -0,0 +1,162 @@
+import json
+from unittest import TestCase
+from search_tags_count import SearchTagsCount
+from elasticsearch import Elasticsearch
+from unittest.mock import patch, MagicMock
+
+
+class TestSearchTagsCount(TestCase):
+    elasticsearch = Elasticsearch(
+        hosts=[{'host': 'localhost'}]
+    )
+
+    def setUp(self):
+        items = [
+            {
+                'article_id': 'test1',
+                'created_at': 1530112710,
+                'title': 'abc1',
+                'published_at': 1530112710,
+                'sort_key': 1530112710000000,
+                'body': 'huga test',
+                'tags': ['A', 'B', 'C']
+            },
+            {
+                'article_id': 'test2',
+                'created_at': 1530112720,
+                'title': 'abc2',
+                'published_at': 1530112710 + 1,
+                'sort_key': 1530112720000000,
+                'body': 'foo bar',
+                'tags': ['A', 'B']
+            },
+            {
+                'article_id': 'test3',
+                'created_at': 1530112730,
+                'title': 'abc3',
+                'published_at': 1530112710 + 2,
+                'sort_key': 1530112730000000,
+                'body': 'foo bar',
+                'tags': ['A']
+            },
+            {
+                'article_id': 'test4',
+                'created_at': 1530112740,
+                'title': 'abc4',
+                'published_at': 1530112710 - 1,
+                'sort_key': 1530112700000000,
+                'body': 'foo bar',
+                'tags': ['A', 'B', 'C']
+            },
+            {
+                'article_id': 'test5',
+                'created_at': 1530112750,
+                'title': 'abc5',
+                'published_at': 1530112710,
+                'sort_key': 1530112700000000,
+                'body': 'foo bar',
+                'tags': ['ﾊﾝｶｸ', '＆＄％！”＃', '𪚲🍣𪚲', 'aaa-aaa', 'abcde vwxyz']
+            },
+        ]
+        for item in items:
+            self.elasticsearch.index(
+                    index="articles",
+                    doc_type="article",
+                    id=item["article_id"],
+                    body=item
+            )
+        self.elasticsearch.indices.refresh(index="articles")
+
+    def tearDown(self):
+        self.elasticsearch.indices.delete(index="articles", ignore=[404])
+
+    @patch('time.time', MagicMock(return_value=1530112710 + 86400 * 7))
+    def test_search_request(self):
+        params = {
+                'queryStringParameters': {
+                    'tags': ['A', 'B', 'C', 'D']
+                }
+        }
+        response = SearchTagsCount(params, {}, elasticsearch=self.elasticsearch).main()
+        actual = json.loads(response['body'])
+        expected = [
+             {'count': 3, 'tag': 'A'},
+             {'count': 2, 'tag': 'B'},
+             {'count': 1, 'tag': 'C'},
+             {'count': 0, 'tag': 'D'}
+        ]
+        self.assertEqual(response['statusCode'], 200)
+        self.assertEqual(expected, actual)
+
+    def test_search_request_not_exists(self):
+        params = {
+                'queryStringParameters': {
+                    'tags': ['あ', 'い', 'う']
+                }
+        }
+        response = SearchTagsCount(params, {}, elasticsearch=self.elasticsearch).main()
+        actual = json.loads(response['body'])
+        expected = [
+            {'count': 0, 'tag': 'あ'},
+            {'count': 0, 'tag': 'い'},
+            {'count': 0, 'tag': 'う'}
+        ]
+        self.assertEqual(response['statusCode'], 200)
+        self.assertEqual(expected, actual)
+
+    @patch('time.time', MagicMock(return_value=1530112710 + 86400 * 7))
+    def test_search_with_tag_half_kana(self):
+        params = {
+                'queryStringParameters': {
+                    'tags': ['ﾊﾝｶｸ', '＆＄％！”＃', '𪚲🍣𪚲', 'aaa-aaa', 'abcde vwxyz']
+                }
+        }
+        response = SearchTagsCount(params, {}, elasticsearch=self.elasticsearch).main()
+        actual = json.loads(response['body'])
+        expected = [
+            {'count': 1, 'tag': 'ﾊﾝｶｸ'},
+            {'count': 1, 'tag': '＆＄％！”＃'},
+            {'count': 1, 'tag': '𪚲🍣𪚲'},
+            {'count': 1, 'tag': 'aaa-aaa'},
+            {'count': 1, 'tag': 'abcde vwxyz'}
+        ]
+        self.assertEqual(response['statusCode'], 200)
+        self.assertEqual(expected, actual)
+
+    def test_search_request_tags_over150(self):
+        # tags 数150超
+        temp_tags = []
+        for i in range(151):
+            temp_tags.append('AAA')
+
+        params = {
+                'queryStringParameters': {
+                    'tags': temp_tags,
+                }
+        }
+        response = SearchTagsCount(params, {}, elasticsearch=self.elasticsearch).main()
+        self.assertEqual(response['statusCode'], 400)
+
+    def test_search_no_params(self):
+        params = {
+            'queryStringParameters': {}
+        }
+        response = SearchTagsCount(params, {}, elasticsearch=self.elasticsearch).main()
+        self.assertEqual(response['statusCode'], 400)
+
+    def test_invalid_tag_parmas(self):
+        params = {
+            'queryStringParameters': {
+                'tags': ['']
+            }
+        }
+        response = SearchTagsCount(params, {}, elasticsearch=self.elasticsearch).main()
+        self.assertEqual(response['statusCode'], 400)
+
+        params = {
+            'queryStringParameters': {
+                'tags': ['A' * 26]
+            }
+        }
+        response = SearchTagsCount(params, {}, elasticsearch=self.elasticsearch).main()
+        self.assertEqual(response['statusCode'], 400)