\u8bb0\u5f55\u6211\u662f\u5982\u4f55\u628arsyslog\u505a\u6210docker\u955c\u50cf\uff0c\u83b7\u53d6nginx\u7684accesslog\u5e76\u4e14\u8f6c\u53d1\u5230python\u7684<\/p>\n
nginx\u65e5\u5fd7\u8981\u8bbe\u7f6e\u6210json\u683c\u5f0f\u8f93\u51fa\uff0cnginx.conf\u5982\u4e0b\u56fe\u6240\u793a\uff0c\u8fd9\u4e2a\u53ef\u4ee5\u5728docker\u955c\u50cf\u4e2d\u901a\u8fc7volume\u628anginx.conf\u6302\u8f7d\u8fdb\u53bb\uff0c\u7136\u540e\u628a\/var\/log\/nginx\/access.log\u6302\u8f7d\u5230\u672c\u5730<\/p>\n
user root;\nworker_processes 1;\nerror_log \/var\/log\/nginx\/error.log warn;\npid \/var\/run\/nginx.pid;\nevents {\n worker_connections 1024;\n}\nhttp {\n include \/etc\/nginx\/mime.types;\n default_type application\/octet-stream;\n #log_format main '$remote_addr - $remote_user [$time_local] \"$request\" '\n # '$status $body_bytes_sent \"$http_referer\" '\n # '\"$http_user_agent\" \"$http_x_forwarded_for\"';\n log_format main '{\"time_local\": \"$time_local\", '\n '\"path\": \"$request_uri\", '\n '\"ip\": \"$remote_addr\", '\n '\"time\": \"$time_iso8601\", '\n '\"user_agent\": \"$http_user_agent\", '\n '\"user_id_got\": \"$uid_got\", '\n '\"user_id_set\": \"$uid_set\", '\n '\"remote_user\": \"$remote_user\", '\n '\"request\": \"$request\", '\n '\"status\": \"$status\", '\n '\"body_bytes_sent\": \"$body_bytes_sent\", '\n '\"request_time\": \"$request_time\", '\n '\"http_referrer\": \"$http_referer\" }';\n access_log \/var\/log\/nginx\/access.log main;\n sendfile on;\n #tcp_nopush on;\n keepalive_timeout 65;\n #gzip on;\n include \/etc\/nginx\/conf.d\/*.conf;\n}\n<\/code><\/pre>\n\u5173\u952e\u70b92 rsyslog\u914d\u7f6e\u4e0eDockerfile<\/h1>\n
\u7f16\u5199\u4e00\u4e2a51-nginx-forward.conf\u6587\u4ef6\u653e\u7f6e\u5728\/etc\/rsyslog.d\/\u4e0b\u5373\u53ef<\/p>\n
module(load=\"imfile\")\ninput(type=\"imfile\"\n File=\"\/var\/log\/nginx\/access.log\"\n Tag=\"mywebsite:\")\n# omfwd module for forwarding the logs to another tcp server\nif( $syslogtag == 'mywebsite:') then {\n action(type=\"omfwd\" target=\"python\u670d\u52a1\u5668IP\u5730\u5740\" port=\"6000\" protocol=\"tcp\"\n action.resumeRetryCount=\"100\"\n queue.type=\"linkedList\" queue.size=\"10000\")\n}\n<\/code><\/pre>\n\u6211\u4eec\u53ef\u4ee5\u7528\u4e00\u4e2aDockerfile\u6765\u8fd0\u884crsyslog\uff0cdocker run\u7684\u65f6\u5019\u6ce8\u610f\u65e5\u5fd7\u7684\u6302\u8f7d<\/p>\n
FROM ubuntu:16.04\nRUN apt-get update && apt-get install -y rsyslog; \\\n rm -rf \/var\/lib\/apt\/lists\/*\nADD 51-nginx-forward.conf \/etc\/rsyslog.d\/.\n# RUN cat \/dev\/null> \/var\/log\/mail.log\nCMD service rsyslog start && tail -f \/var\/log\/syslog<\/code><\/pre>\n\u5173\u952e\u70b93 python\u7a0b\u5e8f\u901a\u8fc7tcp\u7684\u65b9\u5f0f\u8bfb\u53d6rsyslog<\/h1>\n
python\u7a0b\u5e8f\u4e0ersyslog\u5efa\u7acbtcp\u8fde\u63a5\uff0c\u53ef\u4ee5\u5b9e\u65f6\u7684\u8fdb\u884c\u6570\u636e\u5e93\u7684\u63d2\u5165\u8bed\u53e5<\/p>\n
import asyncio\nimport json\nimport time\nimport database_init\nclass LogAnalyser:\n def __init__(self):\n pass\n def process(self, str_input):\n # print(str_input)\n str_input = str_input.decode(\"utf-8\", errors=\"ignore\")\n # Add your processing steps here\n # ...\n try:\n # Extract created_at from the log string\n str_splits = str_input.split(\"{\", 1)\n json_text = \"{\" + str_splits[1]\n data = json.loads(json_text)\n created_at = data[\"time\"]\n request_all = data[\"request\"].split(\" \/\", 1)\n http_type = request_all[0]\n path = data[\"path\"]\n request_time = data[\"request_time\"]\n if PREFIX in data[\"path\"]:\n path = data[\"path\"]\n return http_type, path, created_at,request_time # The order is relevant for INSERT query params\n except Exception as e:\n print(\"error in read_rsylog.py,Class LogAnalyser,function process\")\n print(e)\n return None\n@asyncio.coroutine\ndef handle_echo(reader, writer):\n log_filter = LogAnalyser()\n while True:\n line = yield from reader.readline()\n if not line:\n break\n params = log_filter.process(line)\n if params:\n # \u8fdb\u884c\u4e00\u5806\u64cd\u4f5c\uff0c\u4f8b\u5982\u8fdb\u884c\u6570\u636e\u5e93\u7684\u63d2\u5165\n # execute_sql(params=params)\nif __name__ == '__main__':\n CURSOR = database_init.DBConnect().CURSOR\n CONN = database_init.DBConnect().CONN\n PREFIX = database_init.DBConnect().CONFIG[\"TEST_SWAGGER\"][\"PREFIX\"]\n database_init.DBConnect().create_table()\n loop = asyncio.get_event_loop()\n coro = asyncio.start_server(handle_echo, None, 6000, loop=loop)\n server = loop.run_until_complete(coro)\n # Serve requests until Ctrl+C is pressed\n print('Serving on {}'.format(server.sockets[0].getsockname()))\n try:\n loop.run_forever()\n except KeyboardInterrupt:\n pass\n # Close the server\n print(\"Closing the server.\")\n server.close()\n loop.run_until_complete(server.wait_closed())\n loop.close()\n CURSOR.close()\n CONN.close()\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"\u8bb0\u5f55\u6211\u662f\u5982\u4f55\u628arsyslog\u505a\u6210docker\u955c\u50cf\uff0c\u83b7\u53d6nginx\u7684accesslog\u5e76\u4e14\u8f6c\u53d1\u5230python\u7684 […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[4,5],"tags":[],"class_list":["post-588","post","type-post","status-publish","format-standard","hentry","category-k8s","category-python"],"_links":{"self":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/588","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=588"}],"version-history":[{"count":1,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/588\/revisions"}],"predecessor-version":[{"id":875,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/588\/revisions\/875"}],"wp:attachment":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=588"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=588"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=588"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}