2019

README.md

@@ -6,3 +6,5 @@ sa-scripts
 @……@ Enjoy yourself<br>
 博客：http://www.simlinux.com<br>
 By Geekwolf
+
+![赞赏](https://raw.githubusercontent.com/geekwolf/fms/master/doc/images/wxzf.png)

ops-scripts/lua-ngx-log.md

@@ -0,0 +1,117 @@
+#### 1. install softs
+
+```
+wget https://github.com/openresty/lua-nginx-module/archive/v0.10.15.tar.gz
+wget http://tengine.taobao.org/download/tengine-2.3.2.tar.gz
+wget https://github.com/simplresty/ngx_devel_kit/archive/v0.3.1.tar.gz
+wget https://openresty.org/download/openresty-1.15.8.1.tar.gz
+```
+#### 2. install openrestry luajit2 lib
+```
+./configure --prefix=/user/local/openresty-1.15.8.1
+make 
+make install
+ln -s /usr/local/openresty-1.15.8.1/luajit/lib/libluajit-5.1.so.2 /lib64/libluajit-5.1.so.2
+export LUAJIT_LIB=/usr/local/openresty-1.15.8.1/luajit/lib/
+export LUAJIT_INC=/usr/local/openresty-1.15.8.1/luajit/include/luajit-2.1/
+
+```
+#### 3. install tengine
+
+```
+./configure --prefix=/usr/local/tengine/ --with-http_gzip_static_module --add-module=/data/data/softs/lua-nginx-module-0.10.15/ --add-module=/data/data/softs/ngx_devel_kit-0.3.1/
+make
+make install
+```
+
+#### 4. Nginx Configure: log_by_lua_file
+```
+log_by_lua_file /usr/local/tengine/conf/lua/ngx_log.lua;
+```
+
+### 5. ngx_log.lua
+```
+
+
+-- local file_name
+
+-- if ngx.var.file_type == 'audit' then
+--    file_name = '/usr/local/apps/nginx/logs/audit.log'
+-- elseif ngx.var.file_type == 'mapi'then
+--     file_name = '/usr/local/apps/nginx/logs/access.log'
+-- end
+
+
+local file_audit = '/usr/local/apps/nginx/logs/audit.log'
+local file_mapi  = '/usr/local/apps/nginx/logs/access.log'
+
+
+function write_content(file_name, content)
+        local  f = assert(io.open(file_name,'a'))
+        f:write(content)
+        f:close()
+end
+
+function urlDecode(s)
+    if(nil ~= s)
+    then  
+        s = string.gsub(s, '%%(%x%x)', function(h) return string.char(tonumber(h, 16)) end)  
+        return s  
+    end
+end  
+
+function handle_body(s)
+    s = urlDecode(s)
+    if(nil ~= s)
+    then
+        return string.gsub(s,'payMethodInfo=(.-)&','payMethodInfo=***&')
+    end
+end
+
+
+local remote_addr = ngx.var.remote_addr
+local http_x_forwarded_for = ngx.var.http_x_forwarded_for
+local time_local = ngx.var.time_local
+local server_name = ngx.var.server_name
+local request_method = ngx.var.request_method
+local scheme = ngx.var.scheme
+local host = ngx.var.host
+local request_uri = ngx.var.request_uri
+local status = ngx.var.status
+local referer = ngx.var.referer
+local body = handle_body(ngx.var.request_body)
+local http_user_agent = ngx.var.http_user_agent
+local upstream_status = ngx.var.upstream_status
+local request_time = ngx.var.request_time
+local upstream_response_time = ngx.var.upstream_response_time
+local http_host = ngx.var.http_host
+local scheme_http_host_request_uri = ngx.var.scheme..'://'..http_host..request_uri
+local body_bytes_sent = ngx.var.body_bytes_sent
+local http_referer = ngx.var.http_referer
+local upstream_addr = ngx.var.upstream_addr
+
+
+
+-- if ngx.var.file_type == 'audit' then
+--     local extend = string.format('srcip=%s&x_srcip=%s&time="%s"&server=%s&server_ip=%s&method=%s&link="%s://%s%s&status=%s&referer="%s"&post="%s"&user_agent="%s"',remote_addr,http_x_forwarded_for,time_local,server_name,server_addr,request_method,scheme,host,request_uri,status,referer,body,http_user_agent)
+--     if(ngx.req.get_method() == 'POST')
+--     then
+--         write_content(file_audit, extend..'\n')
+--     end
+-- elseif ngx.var.file_type == 'mapi' then
+--     local extend = string.format('{"upstream_status":%s,"request_time":%s,"upstream_response_time":%s,"remote_addr":%s,"time_local":%s,"scheme_http_host_request_uri":%s,"status":%s,"body_bytes_sent":%s,"http_referer":%s,"request_body":%s,"http_user_agent":%s,"http_x_forwarded_for":%s,"upstream_addr":%s}',upstream_status,request_time,upstream_response_time,remote_addr,time_local,scheme_http_host_request_uri,status,body_bytes_sent,http_referer,request_body,http_user_agent,http_x_forwarded_for,upstream_addr)
+--     write_content(file_mapi, extend..'\n')
+-- end
+
+local extend_audit = string.format('srcip=%s&x_srcip=%s&time="%s"&server=%s&server_ip=%s&method=%s&link="%s://%s%s&status=%s&referer="%s"&post="%s"&user_agent="%s"',remote_addr,http_x_forwarded_for,time_local,server_name,server_addr,request_method,scheme,host,request_uri,status,referer,body,http_user_agent)
+local extend_mapi = string.format('{"upstream_status":%s,"request_time":%s,"upstream_response_time":%s,"remote_addr":%s,"time_local":%s,"scheme_http_host_request_uri":%s,"status":%s,"body_bytes_sent":%s,"http_referer":%s,"request_body":%s,"http_user_agent":%s,"http_x_forwarded_for":%s,"upstream_addr":%s}',upstream_status,request_time,upstream_response_time,remote_addr,time_local,scheme_http_host_request_uri,status,body_bytes_sent,http_referer,request_body,http_user_agent,http_x_forwarded_for,upstream_addr)
+
+if(ngx.req.get_method() == 'POST')
+then
+    write_content(file_audit, extend_audit..'\n')
+end
+write_content(file_mapi, extend_mapi..'\n')
+
+```
+#### 6.Lua加载顺序
+![](https://cloud.githubusercontent.com/assets/2137369/15272097/77d1c09e-1a37-11e6-97ef-d9767035fc3e.png)

ops-scripts/migratetoconfluence.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+# @Author: Geekwolf
+# @Date:   2018-05-24 17:58:16
+# @Last Modified by:   Geekwolf
+# @Last Modified time: 2018-05-25 19:50:37
+
+import pymysql
+import collections
+import requests
+import json
+import markdown
+
+
+class DBHelper(object):
+    """docstring for DBHelper"""
+
+    def __init__(self):
+        self.host = '192.168.1.1'
+        self.user = 'wiki'
+        self.password = 'password'
+        self.database = 'db'
+        self.conn = None
+        self.cur = None
+
+    def ConnDB(self):
+        try:
+            self.conn = pymysql.connect(self.host, self.user, self.password, self.database, charset='utf8')
+        except Exception as e:
+            print(str(e))
+            return False
+        self.cur = self.conn.cursor()
+        return True
+
+    def Close(self):
+        if self.conn and self.cur:
+            self.cur.close()
+            self.conn.close()
+        return True
+
+    def Execute(self, sql, params=None):
+        self.ConnDB()
+        try:
+            if self.conn and self.cur:
+                self.cur.execute(sql, params)
+                self.conn.commit()
+        except Exception as e:
+            print(str(e))
+            self.Close()
+            return False
+        return True
+
+    def Select(self, sql, params=None):
+        self.Execute(sql, params)
+        return self.cur.fetchall()
+
+
+class SyncWiki(object):
+    """docstring for SyncWiki"""
+
+    def __init__(self, ):
+
+        # The Space:autotest Key Name
+        self.space = 'ops'
+        self.url = 'http://confluence'
+        self.username = 'geekwolf'
+        self.password = 'geekwolf'
+        self.session = self.GetSession()
+        # self.home_page = '{} Home'.format(self.space)
+        self.home_page='ops'
+        self.headers = {'Content-Type': 'application/json'}
+        self.dbhelper = DBHelper()
+
+    def GetSession(self):
+        session = requests.session()
+        data = {'os_username': self.username, 'os_password': self.password, 'login': 'Log in'}
+        res = session.post(self.url, data)
+        return session
+
+    def MarkdownToHtml(self, content):
+        # convert_url = "{}/rest/api/contentbody/convert/storage".format(self.url)
+        # print(convert_url)
+        # data = {"value": content, "representation": "wiki"}
+        # ret = self.session.post(convert_url, json.dumps(data), headers=self.headers)
+
+        ret = markdown.markdown(content, extensions=['fenced_code', 'codehilite', 'extra', 'abbr', 'attr_list', 'def_list', 'footnotes',
+                                                     'tables', 'smart_strong', 'admonition', 'codehilite', 'headerid', 'meta', 'nl2br', 'sane_lists', 'smarty', 'toc', 'wikilinks'])
+        return ret
+
+    def GetPageId(self, title):
+        '''
+                通过分类名称获取在Confluence中的id
+        '''
+        content_url = '{}/rest/api/content?spaceKey={}&title={}'.format(self.url, self.space, title)
+        data = self.session.get(content_url).json()
+        id = data['results'][0]['id']
+        return id
+
+    def CreatePageMethod(self, id, title, value=None):
+
+        page_url = '{}/rest/api/content'.format(self.url)
+        data = {"type": "page", "ancestors": [{"id": id}], "title": title, "space": {
+            "key": self.space}, "body": {"storage": {"value": value, "representation": "storage"}}}
+        self.session.post(page_url, json.dumps(data), headers=self.headers)
+
+    def CreateTypePage(self):
+        '''
+                创建分类页面(二级分类)
+        '''
+        group_page_url = '{}/rest/api/content'.format(self.url)
+        group_info = self.GetGroupInfo()
+
+        try:
+            for k, v in group_info.items():
+                self.CreatePageMethod(self.GetPageId(self.home_page), k)
+                for i in v:
+                    self.CreatePageMethod(self.GetPageId(k), i)
+            ret = True
+        except Exception as e:
+            print(str(e))
+            ret = False
+        return ret
+
+    def CreatePage(self):
+        '''
+                根据标题和内容创建对应子类的页面
+        '''
+        content = self.GetWiki()
+        for i in content:
+            try:
+                id = self.GetPageId(i[0])
+                title = i[1]
+                value = self.MarkdownToHtml(i[2])
+                self.CreatePageMethod(id, title, value=value)
+                print('{}------{}已经创建'.format(i[0], i[1]))
+            except Exception as e:
+                print('{}------{}创建失败:{}'.format(i[0], i[1], str(e)))
+
+    def GetGroupInfo(self):
+
+        sql = 'select * from wiki_group;'
+        result = self.dbhelper.Select(sql)
+        _group_info = collections.defaultdict(list)
+        _group_dict = dict([(r[0], r[1]) for r in result])
+
+        for r in result:
+            if r[3] is None:
+                _group_info[r[1]] = []
+            else:
+                _group_info[_group_dict[r[3]]].append(r[1])
+        return _group_info
+
+    def GetWiki(self):
+
+        sql = 'SELECT g.name,w.title,w.content from wiki_wiki as w LEFT JOIN wiki_group  as g ON w.group_id = g.id'
+        result = self.dbhelper.Select(sql)
+        return result
+
+
+if __name__ == '__main__':
+
+    ins = SyncWiki()
+    if ins.CreateTypePage():
+        ins.CreatePage()
+

ops-scripts/plogstash/README.md

@@ -0,0 +1,10 @@
+#### 用途
+
+- 基于Redis List日志消息，归档日志文件
+- 解决Logstash(新版本单线程可解决)归档乱序问题
+- 架构: Filebeat->Redis->Plogstash->Files
+#### 用法:
+```
+python3 plogstash.py
+Usage: plogstash.py [start|stop|restart|status]
+```