Skip to content

Commit a514e8d

Browse files
keillerakeillera
authored
Merge pull request #124 from yaasita/elasticsearch4
Elasticsearchの追加(serverless)
2 parents 9faf294 + 8ddda6d commit a514e8d

File tree

26 files changed

+750
-15
lines changed

26 files changed

+750
-15
lines changed

‎.circleci/config.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ jobs:
1111
docker:
1212
- image: circleci/python:3.6.1
1313
- image: bluszcz/bflocalstack-dynamodb-s3
14+
- image: docker.elastic.co/elasticsearch/elasticsearch:6.2.0
15+
environment:
16+
discovery.type: single-node
1417

1518
working_directory: ~/repo
1619

‎Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
FROM amazonlinux:2018.03.0.20180424
22

3+
34
WORKDIR /workdir
45
COPY requirements.txt ./
56
RUN yum install -y gcc python36 python36-devel

‎README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,11 @@ Specify generated Cognito User Pool ARN to SSM.
8989
- See: https://github.com/AlisProject/environment
9090

9191

92-
### Lambda & API Gateway
92+
### Lambda & API Gateway & ElasticSearch
93+
94+
[check your global ip](https://checkip.amazonaws.com/)
95+
9396
```bash
9497
./deploy.sh api
98+
python elasticsearch-setup.py YourGlobalIP
9599
```

‎api-template.yaml‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,76 @@ Resources:
215215
created_at:
216216
type: integer
217217
paths:
218+
/search/articles:
219+
get:
220+
description: "記事検索"
221+
parameters:
222+
- name: "query"
223+
in: "query"
224+
description: "検索ワード"
225+
required: true
226+
type: "string"
227+
- name: "limit"
228+
in: "query"
229+
description: "取得件数"
230+
required: false
231+
type: "integer"
232+
minimum: 1
233+
- name: "page"
234+
in: "query"
235+
description: "ページ"
236+
required: false
237+
type: "integer"
238+
responses:
239+
"200":
240+
description: "検索記事一覧"
241+
schema:
242+
type: array
243+
items:
244+
$ref: '#/definitions/ArticleInfo'
245+
x-amazon-apigateway-integration:
246+
responses:
247+
default:
248+
statusCode: "200"
249+
uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${ElasticSearchSearchArticles.Arn}/invocations
250+
passthroughBehavior: when_no_templates
251+
httpMethod: POST
252+
type: aws_proxy
253+
/search/users:
254+
get:
255+
description: "ユーザー検索"
256+
parameters:
257+
- name: "query"
258+
in: "query"
259+
description: "検索ワード"
260+
required: true
261+
type: "string"
262+
- name: "limit"
263+
in: "query"
264+
description: "取得件数"
265+
required: false
266+
type: "integer"
267+
minimum: 1
268+
- name: "page"
269+
in: "query"
270+
description: "ページ"
271+
required: false
272+
type: "integer"
273+
responses:
274+
"200":
275+
description: "検索ユーザー一覧"
276+
schema:
277+
type: array
278+
items:
279+
$ref: '#/definitions/UserInfo'
280+
x-amazon-apigateway-integration:
281+
responses:
282+
default:
283+
statusCode: "200"
284+
uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${ElasticSearchSearchUsers.Arn}/invocations
285+
passthroughBehavior: when_no_templates
286+
httpMethod: POST
287+
type: aws_proxy
218288
/articles/recent:
219289
get:
220290
description: "最新記事一覧情報を取得"
@@ -1644,3 +1714,67 @@ Resources:
16441714
Path: /me/unread_notification_managers
16451715
Method: put
16461716
RestApiId: !Ref RestApi
1717+
ElasticSearchSearchArticles:
1718+
Type: AWS::Serverless::Function
1719+
Properties:
1720+
Handler: handler.lambda_handler
1721+
Role: !GetAtt LambdaRole.Arn
1722+
CodeUri: ./deploy/search_articles.zip
1723+
Environment:
1724+
Variables:
1725+
ELASTIC_SEARCH_ENDPOINT: !GetAtt ElasticSearchService.DomainEndpoint
1726+
Events:
1727+
Api:
1728+
Type: Api
1729+
Properties:
1730+
Path: /search/articles
1731+
Method: get
1732+
RestApiId: !Ref RestApi
1733+
ElasticSearchSearchUsers:
1734+
Type: AWS::Serverless::Function
1735+
Properties:
1736+
Handler: handler.lambda_handler
1737+
Role: !GetAtt LambdaRole.Arn
1738+
CodeUri: ./deploy/search_users.zip
1739+
Environment:
1740+
Variables:
1741+
ELASTIC_SEARCH_ENDPOINT: !GetAtt ElasticSearchService.DomainEndpoint
1742+
Events:
1743+
Api:
1744+
Type: Api
1745+
Properties:
1746+
Path: /search/users
1747+
Method: get
1748+
RestApiId: !Ref RestApi
1749+
ElasticSearchService:
1750+
Type: "AWS::Elasticsearch::Domain"
1751+
Properties:
1752+
AccessPolicies: !Join
1753+
- ''
1754+
- - '{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": "'
1755+
- !GetAtt LambdaRole.Arn
1756+
- '" }, "Action": "es:*", "Resource": "'
1757+
- 'arn:aws:es:'
1758+
- !Ref 'AWS::Region'
1759+
- ':'
1760+
- !Ref 'AWS::AccountId'
1761+
- ':domain/'
1762+
- !Ref "AWS::StackName"
1763+
- '/*" } ] }'
1764+
AdvancedOptions:
1765+
rest.action.multi.allow_explicit_index: 'true'
1766+
DomainName: !Ref "AWS::StackName"
1767+
EBSOptions:
1768+
EBSEnabled: true
1769+
VolumeType: gp2
1770+
VolumeSize: 20
1771+
ElasticsearchClusterConfig:
1772+
InstanceType: m4.large.elasticsearch
1773+
InstanceCount: 1
1774+
DedicatedMasterEnabled: true
1775+
ZoneAwarenessEnabled: false
1776+
DedicatedMasterType: m4.large.elasticsearch
1777+
DedicatedMasterCount: 3
1778+
ElasticsearchVersion: '6.2'
1779+
SnapshotOptions:
1780+
AutomatedSnapshotStartHour: 0

‎database-template.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ Resources:
2929
AttributeType: S
3030
- AttributeName: sort_key
3131
AttributeType: N
32+
- AttributeName: sync_elasticsearch
33+
AttributeType: N
3234
KeySchema:
3335
- AttributeName: article_id
3436
KeyType: HASH
@@ -66,6 +68,15 @@ Resources:
6668
ProvisionedThroughput:
6769
ReadCapacityUnits: !Ref MinDynamoReadCapacitty
6870
WriteCapacityUnits: !Ref MinDynamoWriteCapacitty
71+
- IndexName: sync_elasticsearch-index
72+
KeySchema:
73+
- AttributeName: sync_elasticsearch
74+
KeyType: HASH
75+
Projection:
76+
ProjectionType: ALL
77+
ProvisionedThroughput:
78+
ReadCapacityUnits: !Ref MinDynamoReadCapacitty
79+
WriteCapacityUnits: !Ref MinDynamoWriteCapacitty
6980
ProvisionedThroughput:
7081
ReadCapacityUnits: !Ref MinDynamoReadCapacitty
7182
WriteCapacityUnits: !Ref MinDynamoWriteCapacitty
@@ -332,9 +343,21 @@ Resources:
332343
AttributeDefinitions:
333344
- AttributeName: user_id
334345
AttributeType: S
346+
- AttributeName: sync_elasticsearch
347+
AttributeType: N
335348
KeySchema:
336349
- AttributeName: user_id
337350
KeyType: HASH
351+
GlobalSecondaryIndexes:
352+
- IndexName: sync_elasticsearch-index
353+
KeySchema:
354+
- AttributeName: sync_elasticsearch
355+
KeyType: HASH
356+
Projection:
357+
ProjectionType: ALL
358+
ProvisionedThroughput:
359+
ReadCapacityUnits: !Ref MinDynamoReadCapacitty
360+
WriteCapacityUnits: !Ref MinDynamoWriteCapacitty
338361
ProvisionedThroughput:
339362
ReadCapacityUnits: !Ref MinDynamoReadCapacitty
340363
WriteCapacityUnits: !Ref MinDynamoWriteCapacitty

‎database.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ Resources:
1111
AttributeType: S
1212
- AttributeName: sort_key
1313
AttributeType: N
14+
- AttributeName: sync_elasticsearch
15+
AttributeType: N
1416
KeySchema:
1517
- AttributeName: article_id
1618
KeyType: HASH
@@ -48,6 +50,15 @@ Resources:
4850
ProvisionedThroughput:
4951
ReadCapacityUnits: 2
5052
WriteCapacityUnits: 2
53+
- IndexName: sync_elasticsearch-index
54+
KeySchema:
55+
- AttributeName: sync_elasticsearch
56+
KeyType: HASH
57+
Projection:
58+
ProjectionType: ALL
59+
ProvisionedThroughput:
60+
ReadCapacityUnits: 2
61+
WriteCapacityUnits: 2
5162
ProvisionedThroughput:
5263
ReadCapacityUnits: 2
5364
WriteCapacityUnits: 2
@@ -305,9 +316,21 @@ Resources:
305316
AttributeDefinitions:
306317
- AttributeName: user_id
307318
AttributeType: S
319+
- AttributeName: sync_elasticsearch
320+
AttributeType: N
308321
KeySchema:
309322
- AttributeName: user_id
310323
KeyType: HASH
324+
GlobalSecondaryIndexes:
325+
- IndexName: sync_elasticsearch-index
326+
KeySchema:
327+
- AttributeName: sync_elasticsearch
328+
KeyType: HASH
329+
Projection:
330+
ProjectionType: ALL
331+
ProvisionedThroughput:
332+
ReadCapacityUnits: 2
333+
WriteCapacityUnits: 2
311334
ProvisionedThroughput:
312335
ReadCapacityUnits: 2
313336
WriteCapacityUnits: 2

‎elasticsearch-setup.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#!/usr/bin/env python
2+
import boto3
3+
import json
4+
import os
5+
import urllib
6+
import time
7+
import sys
8+
9+
10+
class ESconfig:
11+
def __init__(self):
12+
self.domain = os.environ["ALIS_APP_ID"] + 'api'
13+
self.client = boto3.client('es')
14+
response = self.client.describe_elasticsearch_domain(
15+
DomainName=self.domain
16+
)
17+
self.original_access_policy = response['DomainStatus']['AccessPolicies']
18+
self.arn = json.loads(self.original_access_policy)['Statement'][0]['Resource']
19+
self.endpoint = response['DomainStatus']['Endpoint']
20+
21+
def set_access_policy_allow_ip(self, ip):
22+
new_access_policy = {
23+
"Version": "2012-10-17",
24+
"Statement": [
25+
{
26+
"Effect": "Allow",
27+
"Principal": {
28+
"AWS": "*"
29+
},
30+
"Action": [
31+
"es:*"
32+
],
33+
"Condition": {
34+
"IpAddress": {
35+
"aws:SourceIp": [
36+
ip
37+
]
38+
}
39+
},
40+
"Resource": self.arn
41+
}
42+
]
43+
}
44+
self.client.update_elasticsearch_domain_config(
45+
DomainName=self.domain,
46+
AccessPolicies=json.dumps(new_access_policy)
47+
)
48+
49+
def rollback_access_policy(self):
50+
self.client.update_elasticsearch_domain_config(
51+
DomainName=self.domain,
52+
AccessPolicies=self.original_access_policy
53+
)
54+
55+
56+
esconfig = ESconfig()
57+
myip = sys.argv[1]
58+
esconfig.set_access_policy_allow_ip(myip)
59+
print(myip)
60+
print("アクセスポリシー反映中")
61+
time.sleep(60)
62+
print("インデックス作成")
63+
requst_json = {
64+
"settings": {
65+
"analysis": {
66+
"analyzer": {
67+
"my_ja_analyzer": {
68+
"type": "custom",
69+
"tokenizer": "kuromoji_tokenizer",
70+
"char_filter": [
71+
"icu_normalizer",
72+
"kuromoji_iteration_mark"
73+
],
74+
"filter": [
75+
"kuromoji_baseform",
76+
"kuromoji_part_of_speech",
77+
"ja_stop",
78+
"kuromoji_number",
79+
"kuromoji_stemmer"
80+
]
81+
}
82+
}
83+
}
84+
}
85+
}
86+
87+
index_list = ["articles", "users"]
88+
for index in index_list:
89+
url = f"https://{esconfig.endpoint}/{index}"
90+
request = urllib.request.Request(
91+
url,
92+
method="PUT",
93+
data=json.dumps(requst_json).encode("utf-8"),
94+
headers={"Content-Type": "application/json"}
95+
)
96+
with urllib.request.urlopen(request) as response:
97+
response_body = response.read().decode("utf-8")
98+
print(f"{index}インデックス作成完了")
99+
100+
esconfig.rollback_access_policy()

‎requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,5 @@ rfc3987
66
jinja2
77
pillow
88
aws-requests-auth
9+
requests_aws4auth
10+
elasticsearch

0 commit comments

Comments
 (0)