{"id":1540,"date":"2024-10-24T07:32:00","date_gmt":"2024-10-23T23:32:00","guid":{"rendered":"https:\/\/blog.laoyulaoyu.top\/?p=1540"},"modified":"2024-09-29T18:44:31","modified_gmt":"2024-09-29T10:44:31","slug":"%e8%a7%a3%e5%af%86%ef%bc%81%e6%b7%b1%e5%ba%a6%e5%bc%ba%e5%8c%96%e5%ad%a6%e4%b9%a0%e5%a6%82%e4%bd%95%e5%ae%9e%e7%8e%b0%e8%87%aa%e5%8a%a8%e8%82%a1%e7%a5%a8%e4%ba%a4%e6%98%93","status":"publish","type":"post","link":"https:\/\/laoyulaoyu.com\/index.php\/2024\/10\/24\/%e8%a7%a3%e5%af%86%ef%bc%81%e6%b7%b1%e5%ba%a6%e5%bc%ba%e5%8c%96%e5%ad%a6%e4%b9%a0%e5%a6%82%e4%bd%95%e5%ae%9e%e7%8e%b0%e8%87%aa%e5%8a%a8%e8%82%a1%e7%a5%a8%e4%ba%a4%e6%98%93\/","title":{"rendered":"\u89e3\u5bc6\uff01\u6df1\u5ea6\u5f3a\u5316\u5b66\u4e60\u5982\u4f55\u5b9e\u73b0\u81ea\u52a8\u80a1\u7968\u4ea4\u6613"},"content":{"rendered":"\n<p>\u4f5c\u8005\uff1a<a href=\"https:\/\/www.laoyulaoyu.com\/\" target=\"_blank\" rel=\"noreferrer noopener\">\u8001\u4f59\u635e\u9c7c<\/a><\/p>\n\n\n\n<p><strong><mark style=\"background-color:rgba(0, 0, 0, 0)\" class=\"has-inline-color has-cyan-bluish-gray-color\">\u539f\u521b\u4e0d\u6613\uff0c\u8f6c\u8f7d\u8bf7\u6807\u660e\u51fa\u5904\u53ca\u539f\u4f5c\u8005\u3002<\/mark><\/strong><\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" src=\"https:\/\/www.laoyulaoyu.com\/wp-content\/uploads\/2024\/09\/1727579761105.png\" alt=\"\" class=\"wp-image-2273\"\/><\/figure>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\">\n<pre class=\"wp-block-verse\"><strong>\u5199\u5728\u524d\u9762\u7684\u8bdd\uff1a<\/strong>\u672c\u6587\u4ecb\u7ecd\u4e86<mark style=\"background-color:rgba(0, 0, 0, 0)\" class=\"has-inline-color has-vivid-cyan-blue-color\">\u5229\u7528\u6df1\u5ea6\u5f3a\u5316\u5b66\u4e60\uff08DRL\uff09\u521b\u5efa\u81ea\u52a8\u80a1\u7968\u4ea4\u6613\u7cfb\u7edf\u7684\u7814\u7a76\u4e0e\u5b9e\u73b0\u8fc7\u7a0b<\/mark>\u3002\u6587\u4e2d\u8be6\u7ec6\u9610\u8ff0\u4e86\u5982\u4f55\u5229\u7528 Python \u5e93\u83b7\u53d6\u6570\u636e\uff0c\u589e\u5f3a\u6280\u672f\u6307\u6807\uff0c\u7528\u4e0d\u540c DRL \u7b97\u6cd5\u8bad\u7ec3\u591a\u4e2a\u4ee3\u7406\uff0c\u8bbe\u8ba1\u96c6\u5408\u4ee3\u7406\u63d0\u9ad8\u6027\u80fd\uff0c\u5b9a\u4e49\u56de\u8c03\u51fd\u6570\u76d1\u63a7\u8bad\u7ec3\uff0c\u8bc4\u4f30\u4ee3\u7406\u8868\u73b0\u786e\u5b9a\u6700\u4f73\u7b56\u7565\uff0c\u672c\u6587\u8fd8\u5c55\u793a\u5982\u4f55\u7528\u6a21\u578b\u63d0\u4f9b\u4ea4\u6613\u5efa\u8bae\u3002<\/pre>\n<\/blockquote>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"f157\"><strong>\u4e00\u3001\u53c2\u8003\u8d44\u6599<\/strong><\/h2>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u200fYang\uff0cHongyang\uff0cet al. &#8220;<a href=\"https:\/\/papers.ssrn.com\/sol3\/papers.cfm?abstract_id=3690996\">Deep reinforcement learning for automated stock trading: An ensemble strategy<\/a>&#8220;\u3002\u9996\u5c4a ACM \u91d1\u878d\u4eba\u5de5\u667a\u80fd\u56fd\u9645\u4f1a\u8bae\u8bba\u6587\u96c6\u30022020.<\/li>\n\n\n\n<li><a href=\"https:\/\/github.com\/theanh97\/Deep-Reinforcement-Learning-with-Stock-Trading\">https:\/\/github.com\/theanh97\/Deep-Reinforcement-Learning-with-Stock-Trading<\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/medium.com\/@pta.forwork\/deep-reinforcement-learning-for-automated-stock-trading-9d47457707fa\">https:\/\/medium.com\/@pta.forwork\/deep-reinforcement-learning-for-automated-stock-trading-9d47457707fa<\/a><\/li>\n<\/ol>\n\n\n\n<p>\u672c\u6587\u53d7\u4e0a\u8ff0 &#8220;\u53c2\u8003\u8d44\u6599 1 &#8220;\u7814\u7a76\u8bba\u6587\u7684\u542f\u53d1\u3002\u5b83\u8fd8\u53d7\u5230 &#8220;\u53c2\u8003\u8d44\u6599 2 \u548c 3 &#8220;\u7684\u5f71\u54cd\uff0c\u5c06\u5b83\u4eec\u4f5c\u4e3a\u6280\u672f\u57fa\u51c6\uff0c\u5e76\u5bf9\u5176\u4e2d\u4ecb\u7ecd\u7684\u5de5\u4f5c\u6dfb\u52a0\u4e86\u4e00\u4e9b\u4fee\u6b63\u548c\u6539\u8fdb\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><strong>\u4e8c\u3001\u6982\u8ff0<\/strong><\/h2>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" src=\"https:\/\/www.laoyulaoyu.com\/wp-content\/uploads\/2024\/09\/image-237.png\" alt=\"\" class=\"wp-image-2276\"\/><\/figure>\n\n\n\n<p>\u672c\u6587\u65e8\u5728\u8bf4\u660e\u5982\u4f55\u5229\u7528\u6df1\u5ea6\u5f3a\u5316\u5b66\u4e60\uff08DRL\uff09\u521b\u5efa\u4e00\u4e2a\u81ea\u52a8\u80a1\u7968\u4ea4\u6613\u7cfb\u7edf\u3002\u901a\u8fc7\u5e94\u7528\u5148\u8fdb\u7684\u673a\u5668\u5b66\u4e60\u6280\u672f\uff0c\u8be5\u7cfb\u7edf\u5c06\u88ab\u8bad\u7ec3\u6210\u80fd\u5728\u80a1\u5e02\u4e2d\u505a\u51fa\u6709\u5229\u53ef\u56fe\u7684\u4ea4\u6613\u51b3\u7b56\u3002\u8be5\u9879\u76ee\u5305\u62ec\u4e0b\u8f7d\u5386\u53f2\u80a1\u7968\u6570\u636e\uff0c\u7528\u6280\u672f\u6307\u6807\u5bf9\u5176\u8fdb\u884c\u589e\u5f3a\uff0c\u5f00\u53d1\u6a21\u62df\u4ea4\u6613\u73af\u5883\uff0c\u4ee5\u53ca\u8bad\u7ec3\u591a\u4e2a DRL \u4ee3\u7406\uff0c\u5982 PPO\u3001A2C\u3001DDPG\u3001SAC\u3001TD3 \u548c\u4e00\u4e2a\u96c6\u5408\u4ee3\u7406\u3002\u5c06\u4f7f\u7528\u5404\u79cd\u91d1\u878d\u6307\u6807\u5bf9\u8fd9\u4e9b\u4ee3\u7406\u7684\u6027\u80fd\u8fdb\u884c\u8bc4\u4f30\u548c\u6bd4\u8f83\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"e3cc\"><strong>\u4e09\u3001\u6df1\u5ea6\u5f3a\u5316\u5b66\u4e60\uff08DRL\uff09<\/strong><\/h2>\n\n\n\n<p id=\"df8f\">\u5f3a\u5316\u5b66\u4e60\u80cc\u666f\u4e0b\u7684\u6df1\u5ea6\u5b66\u4e60\u6d89\u53ca\u4e00\u4e2a\u4ee3\u7406\u4e0e\u73af\u5883\u4ea4\u4e92\uff0c\u901a\u8fc7\u8bd5\u9519\u5b66\u4e60\u6700\u4f73\u884c\u4e3a\u3002\u8be5\u9886\u57df\u7684\u4e3b\u8981\u672f\u8bed\u548c\u6982\u5ff5\u5bf9\u4e8e\u7406\u89e3\u8fd9\u4e9b\u7cfb\u7edf\u5982\u4f55\u8fd0\u884c\u81f3\u5173\u91cd\u8981\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u73af\u5883\uff1a<\/strong>\u4e0e\u4ee3\u7406\u4e92\u52a8\u7684\u5916\u90e8\u7cfb\u7edf\u3002\u5b83\u63d0\u4f9b\u4e86\u4e00\u4e2a\u53ef\u4ee5\u91c7\u53d6\u884c\u52a8\u5e76\u83b7\u5f97\u54cd\u5e94\uff08\u4ee5\u89c2\u5bdf\u548c\u5956\u52b1\u7684\u5f62\u5f0f\uff09\u7684\u73af\u5883\u3002\u4f8b\u5982\uff0c\u6e38\u620f\u4e16\u754c\u3001\u673a\u5668\u4eba\u7cfb\u7edf\u6216\u4efb\u4f55\u8fdb\u884c\u5b66\u4e60\u7684\u6a21\u62df\u573a\u666f\u3002<\/li>\n\n\n\n<li><strong>\u72b6\u6001\uff1a<\/strong>\u5f53\u524d\u60c5\u51b5\u6216\u73af\u5883\u914d\u7f6e\u7684\u8868\u5f81\u3002\u72b6\u6001\u5305\u542b\u51b3\u7b56\u6240\u9700\u7684\u6240\u6709\u5fc5\u8981\u4fe1\u606f\u3002\u5728\u6e38\u620f\u4e2d\uff0c\u72b6\u6001\u53ef\u4ee5\u5305\u62ec\u6240\u6709\u89d2\u8272\u548c\u7269\u4f53\u7684\u4f4d\u7f6e\u3002<\/li>\n\n\n\n<li><strong>\u89c2\u6d4b\u6570\u636e\uff1a<\/strong>\u4ee3\u7406\u4ece\u73af\u5883\u4e2d\u611f\u77e5\u5230\u7684\u6570\u636e\u3002\u89c2\u6d4b\u6570\u636e\u53ef\u4ee5\u662f\u72b6\u6001\u7684\u90e8\u5206\u6216\u5b8c\u6574\u8868\u8ff0\u3002\u5728\u8bb8\u591a\u60c5\u51b5\u4e0b\uff0c\u4ee3\u7406\u65e0\u6cd5\u83b7\u5f97\u5b8c\u6574\u7684\u72b6\u6001\uff0c\u5fc5\u987b\u4f9d\u9760\u89c2\u5bdf\u7ed3\u679c\u6765\u63a8\u65ad\u3002<\/li>\n\n\n\n<li><strong>\u884c\u52a8\uff1a<\/strong>\u4ee3\u7406\u9488\u5bf9\u5f53\u524d\u72b6\u6001\u6216\u89c2\u5bdf\u7ed3\u679c\u505a\u51fa\u7684\u51b3\u5b9a\u6216\u884c\u52a8\u3002\u884c\u52a8\u4f1a\u6539\u53d8\u73af\u5883\u7684\u72b6\u6001\u3002\u4ee3\u7406\u53ef\u4ee5\u91c7\u53d6\u7684\u6240\u6709\u53ef\u80fd\u884c\u52a8\u7684\u96c6\u5408\u79f0\u4e3a\u884c\u52a8\u7a7a\u95f4\u3002<\/li>\n\n\n\n<li><strong>\u6b65\u9aa4\uff1a<\/strong>\u4ee3\u7406\u4e0e\u73af\u5883\u4ea4\u4e92\u5faa\u73af\u4e2d\u7684\u4e00\u6b21\u8fed\u4ee3\u3002\u5728\u4e00\u4e2a\u6b65\u9aa4\u4e2d\uff0c\u4ee3\u7406\u6839\u636e\u5176\u5f53\u524d\u7b56\u7565\u91c7\u53d6\u4e00\u9879\u884c\u52a8\uff0c\u4ece\u73af\u5883\u4e2d\u63a5\u6536\u89c2\u5bdf\u7ed3\u679c\u548c\u5956\u52b1\uff0c\u5e76\u8fc7\u6e21\u5230\u4e00\u4e2a\u65b0\u7684\u72b6\u6001\u3002<\/li>\n\n\n\n<li><strong>\u7b56\u7565\uff1a<\/strong>\u4e00\u79cd\u7b56\u7565\uff0c\u7531\u4ee3\u7406\u6839\u636e\u72b6\u6001\u6216\u89c2\u5bdf\u7ed3\u679c\u51b3\u5b9a\u91c7\u53d6\u4f55\u79cd\u884c\u52a8\u3002\u7b56\u7565\u53ef\u4ee5\u662f\u786e\u5b9a\u6027\u7684\uff08\u5bf9\u7ed9\u5b9a\u72b6\u6001\u603b\u662f\u91c7\u53d6\u76f8\u540c\u7684\u884c\u52a8\uff09\uff0c\u4e5f\u53ef\u4ee5\u662f\u968f\u673a\u6027\u7684\uff08\u6839\u636e\u6982\u7387\u5206\u5e03\u9009\u62e9\u884c\u52a8\uff09\u3002<\/li>\n\n\n\n<li><strong>\u5956\u52b1\uff1a<\/strong>\u4ee3\u7406\u5728\u91c7\u53d6\u67d0\u9879\u884c\u52a8\u540e\u6536\u5230\u7684\u6807\u91cf\u53cd\u9988\u4fe1\u53f7\u3002\u5956\u52b1\u91cf\u5316\u4e86\u884c\u52a8\u5e26\u6765\u7684\u76f4\u63a5\u6536\u76ca\uff0c\u7528\u4e8e\u5f3a\u5316\u7406\u60f3\u884c\u4e3a\u3002\u4ee3\u7406\u7684\u76ee\u6807\u662f\u5728\u4e00\u6bb5\u65f6\u95f4\u5185\u4f7f\u7d2f\u79ef\u5956\u52b1\u6700\u5927\u5316\u3002<\/li>\n\n\n\n<li><strong>\u96c6<\/strong>: \u4ece\u521d\u59cb\u72b6\u6001\u5230\u7ec8\u7aef\u72b6\u6001\u7684\u4e00\u7cfb\u5217\u6b65\u9aa4\u3002\u5f53\u8fbe\u5230\u9884\u5b9a\u6761\u4ef6\uff08\u5982\u8fbe\u5230\u76ee\u6807\u6216\u65f6\u95f4\u8017\u5c3d\uff09\u65f6\uff0c\u4e00\u96c6\u7ed3\u675f\u3002<\/li>\n<\/ul>\n\n\n\n<p id=\"16c4\">\u5728\u5f3a\u5316\u5b66\u4e60\u4e2d\uff0c\u4ee3\u7406\u7684\u76ee\u6807\u662f\u901a\u8fc7\u53cd\u590d\u4e0e\u73af\u5883\u4e92\u52a8\u3001\u91c7\u53d6\u884c\u52a8\uff0c\u5e76\u6839\u636e\u6240\u83b7\u5956\u52b1\u8c03\u6574\u5176\u7b56\u7565\uff0c\u4ece\u800c\u5b66\u4e60\u4e00\u79cd\u80fd\u4f7f\u7d2f\u79ef\u5956\u52b1\u6700\u5927\u5316\u7684\u7b56\u7565\u3002\u8fd9\u4e9b\u8981\u7d20\u7684\u7ed3\u5408\u4f7f\u4ee3\u7406\u80fd\u591f\u53d1\u5c55\u51fa\u590d\u6742\u7684\u884c\u4e3a\uff0c\u5e76\u901a\u8fc7\u7ecf\u9a8c\u63d0\u9ad8\u5176\u6027\u80fd\u3002<\/p>\n\n\n\n<p id=\"1992\">\u4e0b\u9762\u7b80\u8981\u4ecb\u7ecd\u5e38\u89c1\u7684\u5f3a\u5316\u5b66\u4e60\u7b97\u6cd5 PPO\u3001A2C\u3001DDPG\u3001SAC \u548c TD3\uff08\u53c8\u79f0 &#8220;DRL \u4ee3\u7406&#8221;\uff09\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image\" id=\"990a\"><img decoding=\"async\" src=\"https:\/\/cdn-images-1.readmedium.com\/v2\/resize:fit:800\/1*tHWKv2Znjx9gWLkCJuJABw.png\" alt=\"\"\/><\/figure>\n\n\n\n<p><\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"89a5\"><strong>\u56db\u3001\u4ee3\u7801\u5b9e\u65bd<\/strong><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><br><strong>4.1 \u6570\u636e\u6536\u96c6\u4e0e\u6574\u7406<\/strong><\/h3>\n\n\n\n<p id=\"f087\">\u8981\u5f00\u59cb\u6df1\u5ea6\u5f3a\u5316\u5b66\u4e60\u81ea\u52a8\u80a1\u7968\u4ea4\u6613\u4e4b\u65c5\uff0c\u6211\u4eec\u9996\u5148\u9700\u8981\u6536\u96c6\u5fc5\u8981\u7684\u6570\u636e\u3002\u7b2c\u4e00\u6b65\u662f\u52a0\u8f7d\u5386\u53f2\u80a1\u7968\u6570\u636e\uff0c\u4f5c\u4e3a\u6211\u4eec\u4ea4\u6613\u6a21\u578b\u7684\u57fa\u7840\u3002<\/p>\n\n\n\n<p id=\"b392\">\u5728\u672c\u9879\u76ee\u4e2d\uff0c\u6211\u4eec\u4f7f\u7528 numpy\u3001pandas \u548c yfinance \u7b49 Python \u5e93\u6765\u83b7\u53d6\u548c\u5904\u7406\u80a1\u7968\u5e02\u573a\u6570\u636e\u3002\u5177\u4f53\u6765\u8bf4\uff0c\u6211\u4eec\u91cd\u70b9\u5173\u6ce8\u9053\u743c\u65af 30 \u6307\u6570\uff0c\u8fd9\u662f\u4e00\u4efd\u7531 30 \u53ea\u8457\u540d\u80a1\u7968\u7ec4\u6210\u7684\u5217\u8868\u3002\u6211\u4eec\u5229\u7528\u96c5\u864e\u8d22\u7ecf\u4e0b\u8f7d\u8fd9\u4e9b\u80a1\u7968\u7684\u5386\u53f2\u6570\u636e\uff0c\u65f6\u95f4\u8de8\u5ea6\u4e3a 2009 \u5e74 1 \u6708 1 \u65e5\u81f3 2020 \u5e74 5 \u6708 8 \u65e5\u3002\u8fd9\u4e9b\u6570\u636e\u5bf9\u4e8e\u8bad\u7ec3\u548c\u6d4b\u8bd5\u6211\u4eec\u7684\u5f3a\u5316\u5b66\u4e60\u6a21\u578b\u81f3\u5173\u91cd\u8981\u3002\u901a\u8fc7\u521b\u5efa\u5b57\u5178\u6765\u5b58\u50a8\u8fd9\u4e9b\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u786e\u4fdd\u5728\u6574\u4e2a\u9879\u76ee\u4e2d\u9ad8\u6548\u3001\u6709\u5e8f\u5730\u8bbf\u95ee\u8fd9\u4e9b\u6570\u636e\u3002\u901a\u8fc7\u8fd9\u79cd\u8bbe\u7f6e\uff0c\u6211\u4eec\u53ef\u4ee5\u5728\u5c06\u80a1\u7968\u6570\u636e\u8f93\u5165\u6211\u4eec\u7684\u4ea4\u6613\u7b97\u6cd5\u4e4b\u524d\u5bf9\u5176\u8fdb\u884c\u5206\u6790\u548c\u9884\u5904\u7406\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code>import numpy as np\nimport pandas as pd\nimport yfinance as yf\nimport gymnasium as gym\nfrom gymnasium import spaces\nimport matplotlib.pyplot as plt\nfrom stable_baselines3 import PPO, A2C, DDPG, SAC, TD3\nfrom stable_baselines3.common.vec_env import DummyVecEnv\nfrom stable_baselines3.common.callbacks import BaseCallback\n\n# List of stocks in the Dow Jones 30\ntickers = &#91;\n    'MMM', 'AXP', 'AAPL', 'BA', 'CAT', 'CVX', 'CSCO', 'KO', 'DIS', 'DOW',\n    'GS', 'HD', 'IBM', 'INTC', 'JNJ', 'JPM', 'MCD', 'MRK', 'MSFT', 'NKE',\n    'PFE', 'PG', 'TRV', 'UNH', 'UTX', 'VZ', 'V', 'WBA', 'WMT', 'XOM'\n]\ntickers.remove('DOW')\ntickers.remove('UTX')\n\n# Get historical data from Yahoo Finance and save it to dictionary\ndef fetch_stock_data(tickers, start_date, end_date):\n    stock_data = {}\n    for ticker in tickers:\n        stock_data&#91;ticker] = yf.download(ticker, start=start_date, end=end_date)\n    return stock_data\n\n# Call the function to get data\nstock_data = fetch_stock_data(tickers, '2009-01-01', '2020-05-08')<\/code><\/code><\/pre>\n\n\n\n<p id=\"5b5a\">\u4e3a\u786e\u4fdd\u6a21\u578b\u7684\u7a33\u5065\u6027\u548c\u901a\u7528\u6027\uff0c\u6211\u4eec\u5c06\u5386\u53f2\u80a1\u7968\u6570\u636e\u5206\u4e3a\u4e09\u4e2a\u4e0d\u540c\u7684\u6570\u636e\u96c6\uff1a\u8bad\u7ec3\u6570\u636e\u96c6\u3001\u9a8c\u8bc1\u6570\u636e\u96c6\u548c\u6d4b\u8bd5\u6570\u636e\u96c6\u3002\u8fd9\u6837\uff0c\u6211\u4eec\u5c31\u53ef\u4ee5\u5728\u4e00\u7ec4\u6570\u636e\u4e0a\u8bad\u7ec3\u6a21\u578b\uff0c\u5728\u7b2c\u4e8c\u7ec4\u6570\u636e\u4e0a\u9a8c\u8bc1\u5176\u6027\u80fd\uff0c\u6700\u540e\u5728\u7b2c\u4e09\u7ec4\u6570\u636e\u4e0a\u6d4b\u8bd5\u5176\u6709\u6548\u6027\uff0c\u786e\u4fdd\u6211\u4eec\u7684\u6a21\u578b\u5728\u672a\u89c1\u6570\u636e\u4e0a\u8868\u73b0\u826f\u597d\u3002<\/p>\n\n\n\n<p id=\"3d45\">\u5728\u672c\u9879\u76ee\u4e2d\uff0c\u6211\u4eec\u6307\u5b9a 2009 \u5e74 1 \u6708 1 \u65e5\u81f3 2015 \u5e74 12 \u6708 31 \u65e5\u4e3a\u8bad\u7ec3\u6570\u636e\u96c6\u3002\u8fd9\u662f\u6700\u5927\u7684\u6570\u636e\u96c6\uff0c\u7528\u4e8e\u8bad\u7ec3\u6211\u4eec\u7684\u5f3a\u5316\u5b66\u4e60\u6a21\u578b\u3002\u9a8c\u8bc1\u6570\u636e\u96c6\u7684\u65f6\u95f4\u8de8\u5ea6\u4e3a 2016 \u5e74 1 \u6708 1 \u65e5\u81f3 2016 \u5e74 12 \u6708 31 \u65e5\uff0c\u7528\u4e8e\u5fae\u8c03\u6a21\u578b\u548c\u9632\u6b62\u8fc7\u62df\u5408\u3002\u6700\u540e\uff0c\u4ece 2017 \u5e74 1 \u6708 1 \u65e5\u5230 2020 \u5e74 5 \u6708 8 \u65e5\u7684\u6d4b\u8bd5\u6570\u636e\u96c6\u7528\u4e8e\u8bc4\u4f30\u6a21\u578b\u5728\u5b9e\u9645\u573a\u666f\u4e2d\u7684\u6027\u80fd\u3002<\/p>\n\n\n\n<p>\u6211\u4eec\u5c06\u9053\u743c\u65af 30 \u6307\u6570\u4e2d\u6bcf\u53ea\u80a1\u7968\u7684\u6570\u636e\u8fdb\u884c\u76f8\u5e94\u7684\u62c6\u5206\u3002\u901a\u8fc7\u7ed8\u5236\u82f9\u679c\u516c\u53f8 (AAPL) \u5728\u8fd9\u4e09\u4e2a\u65f6\u6bb5\u7684\u5f00\u76d8\u4ef7\uff0c\u6211\u4eec\u53ef\u4ee5\u76f4\u89c2\u5730\u770b\u5230\u6570\u636e\u5206\u5e03\uff0c\u5e76\u786e\u4fdd\u6b63\u786e\u5b9e\u65bd\u5206\u5272\u3002\u8fd9\u79cd\u7ec6\u81f4\u7684\u5206\u5272\u5bf9\u4e8e\u5f00\u53d1\u53ef\u9760\u7684\u81ea\u52a8\u4ea4\u6613\u7cfb\u7edf\u81f3\u5173\u91cd\u8981\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code># split the data into training, validation and test sets\ntraining_data_time_range = ('2009-01-01', '2015-12-31')\nvalidation_data_time_range = ('2016-01-01', '2016-12-31')\ntest_data_time_range = ('2017-01-01', '2020-05-08')\n\n# split the data into training, validation and test sets\ntraining_data = {}\nvalidation_data = {}\ntest_data = {}\n\nfor ticker, df in stock_data.items():\n    training_data&#91;ticker] = df.loc&#91;training_data_time_range&#91;0]:training_data_time_range&#91;1]]\n    validation_data&#91;ticker] = df.loc&#91;validation_data_time_range&#91;0]:validation_data_time_range&#91;1]]\n    test_data&#91;ticker] = df.loc&#91;test_data_time_range&#91;0]:test_data_time_range&#91;1]]\n\n# print shape of training, validation and test data\nticker = 'AAPL'\nprint(f'- Training data shape for {ticker}: {training_data&#91;ticker].shape}')\nprint(f'- Validation data shape for {ticker}: {validation_data&#91;ticker].shape}')\nprint(f'- Test data shape for {ticker}: {test_data&#91;ticker].shape}\\n')\n\n# Display the first 5 rows of the data\ndisplay(stock_data&#91;'AAPL'].head())\nprint('\\n')\n\n# Plot:\nplt.figure(figsize=(12, 4))\nplt.plot(training_data&#91;ticker].index, training_data&#91;ticker]&#91;'Open'], label='Training', color='blue')\nplt.plot(validation_data&#91;ticker].index, validation_data&#91;ticker]&#91;'Open'], label='Validation', color='red')\nplt.plot(test_data&#91;ticker].index, test_data&#91;ticker]&#91;'Open'], label='Test', color='green')\nplt.xlabel('Date')\nplt.ylabel('Value')\nplt.title(f'{ticker} Stock, Open Price')\nplt.legend()\nplt.show()<\/code><\/code><\/pre>\n\n\n\n<p>\u76f8\u5e94\u7684\u7ed3\u679c\u5982\u4e0b\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" src=\"https:\/\/www.laoyulaoyu.com\/wp-content\/uploads\/2024\/09\/image-238.png\" alt=\"\" class=\"wp-image-2278\"\/><\/figure>\n\n\n\n<p><\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><strong>4.2 \u5f00\u53d1\u7b56\u7565<\/strong><\/h3>\n\n\n\n<p id=\"639e\">\u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u7528\u5bf9\u4ea4\u6613\u7b56\u7565\u5f00\u53d1\u81f3\u5173\u91cd\u8981\u7684\u5404\u79cd\u6280\u672f\u6307\u6807\u6765\u4e30\u5bcc\u6211\u4eec\u7684\u6570\u636e\u96c6\u3002\u6211\u4eec\u8ba1\u7b97\u4e86\u51e0\u4e2a\u5173\u952e\u6307\u6807\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>MACD\uff08\u79fb\u52a8\u5e73\u5747\u6536\u655b\u80cc\u79bb\uff09\uff1a<\/strong>\u8be5\u6307\u6807\u901a\u8fc7\u8ba1\u7b97 12 \u5929\u548c 26 \u5929\u6307\u6570\u79fb\u52a8\u5e73\u5747\u7ebf (EMA) \u6765\u786e\u5b9a MACD \u7ebf\uff0c\u7136\u540e\u5c06 9 \u5929 EMA \u5e94\u7528\u4e8e MACD \u7ebf\u4ee5\u751f\u6210\u4fe1\u53f7\u7ebf\u3002MACD \u7ebf\u548c\u4fe1\u53f7\u7ebf\u6709\u52a9\u4e8e\u6839\u636e\u5176\u4ea4\u53c9\u60c5\u51b5\u786e\u5b9a\u6f5c\u5728\u7684\u4e70\u5165\u6216\u5356\u51fa\u4fe1\u53f7\u3002<\/li>\n\n\n\n<li><strong>RSI\uff08\u76f8\u5bf9\u5f3a\u5f31\u6307\u6570\uff09\uff1a<\/strong>\u6211\u4eec\u4ee5 14 \u5929\u4e3a\u7a97\u53e3\u8ba1\u7b97 RSI\uff0c\u4ee5\u8861\u91cf\u4ef7\u683c\u53d8\u52a8\u7684\u52bf\u5934\u3002\u8be5\u6307\u6807\u901a\u8fc7\u8861\u91cf\u4ef7\u683c\u53d8\u52a8\u7684\u901f\u5ea6\u548c\u53d8\u5316\uff0c\u5e2e\u52a9\u8bc6\u522b\u8d85\u4e70\u6216\u8d85\u5356\u60c5\u51b5\u3002<\/li>\n\n\n\n<li><strong>CCI\uff08\u5546\u54c1\u901a\u9053\u6307\u6570\uff09\uff1a<\/strong>\u8be5\u6307\u6807\u8bc4\u4f30\u4ef7\u683c\u4e0e\u5176\u5e73\u5747\u503c\u7684\u504f\u5dee\uff0c\u6709\u52a9\u4e8e\u8bc6\u522b\u65b0\u8d8b\u52bf\u6216\u6781\u7aef\u60c5\u51b5\u3002\u6211\u4eec\u4f7f\u7528 20 \u5929\u7a97\u53e3\u6765\u8ba1\u7b97 CCI\u3002<\/li>\n\n\n\n<li><strong>ADX\uff08\u5e73\u5747\u65b9\u5411\u6307\u6570\uff09\uff1a<\/strong>\u4e3a\u4e86\u8861\u91cf\u8d8b\u52bf\u7684\u5f3a\u5ea6\uff0c\u6211\u4eec\u4f7f\u7528 14 \u5929\u7a97\u53e3\u8ba1\u7b97 ADX\u3002\u8fd9\u5305\u62ec\u8ba1\u7b97\u65b9\u5411\u8fd0\u52a8\uff08DM\uff09\u6307\u6807\u548c\u5e73\u5747\u771f\u5b9e\u8303\u56f4\uff08ATR\uff09\uff0c\u4ee5\u786e\u5b9a ADX \u503c\u3002<\/li>\n<\/ul>\n\n\n\n<p id=\"f327\">\u901a\u8fc7\u6dfb\u52a0\u8fd9\u4e9b\u6307\u6807\uff0c\u6211\u4eec\u5c06\u539f\u59cb\u80a1\u7968\u4ef7\u683c\u6570\u636e\u8f6c\u5316\u4e3a\u4e00\u4e2a\u7279\u5f81\u4e30\u5bcc\u7684\u6570\u636e\u96c6\uff0c\u4ece\u800c\u66f4\u597d\u5730\u6355\u6349\u5e02\u573a\u8d8b\u52bf\u548c\u4ef7\u683c\u52a8\u6001\u3002\u7136\u540e\uff0c\u8fd9\u4e2a\u589e\u5f3a\u578b\u6570\u636e\u96c6\u5c06\u7528\u4e8e\u8bad\u7ec3\u548c\u8bc4\u4f30\u6211\u4eec\u7684\u5f3a\u5316\u5b66\u4e60\u6a21\u578b\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code>def add_technical_indicators(df):\n\n    df = df.copy()\n\n    # Calculate EMA 12 and 26 for MACD\n    df.loc&#91;:, 'EMA12'] = df&#91;'Close'].ewm(span=12, adjust=False).mean()\n    df.loc&#91;:, 'EMA26'] = df&#91;'Close'].ewm(span=26, adjust=False).mean()\n    df.loc&#91;:, 'MACD'] = df&#91;'EMA12'] - df&#91;'EMA26']\n    df.loc&#91;:, 'Signal'] = df&#91;'MACD'].ewm(span=9, adjust=False).mean()\n\n    # Calculate RSI 14\n    rsi_14_mode = True\n    delta = df&#91;'Close'].diff()\n    if rsi_14_mode:\n        gain = (delta.where(delta &gt; 0, 0)).rolling(window=14).mean()\n        loss = (-delta.where(delta &lt; 0, 0)).rolling(window=14).mean()\n        rs = gain \/ loss\n    else:\n        up = delta.where(delta &gt; 0, 0)\n        down = -delta.where(delta &lt; 0, 0)\n        rs = up.rolling(window=14).mean() \/ down.rolling(window=14).mean()\n    df.loc&#91;:, 'RSI'] = 100 - (100 \/ (1 + rs))\n\n    # Calculate CCI 20\n    tp = (df&#91;'High'] + df&#91;'Low'] + df&#91;'Close']) \/ 3\n    sma_tp = tp.rolling(window=20).mean()\n    mean_dev = tp.rolling(window=20).apply(lambda x: np.mean(np.abs(x - x.mean())))\n    df.loc&#91;:, 'CCI'] = (tp - sma_tp) \/ (0.015 * mean_dev)\n\n    # Calculate ADX 14\n    high_diff = df&#91;'High'].diff()\n    low_diff = df&#91;'Low'].diff()\n    df.loc&#91;:, '+DM'] = np.where((high_diff &gt; low_diff) &amp; (high_diff &gt; 0), high_diff, 0)\n    df.loc&#91;:, '-DM'] = np.where((low_diff &gt; high_diff) &amp; (low_diff &gt; 0), low_diff, 0)\n    tr = pd.concat(&#91;df&#91;'High'] - df&#91;'Low'], np.abs(df&#91;'High'] - df&#91;'Close'].shift(1)), np.abs(df&#91;'Low'] - df&#91;'Close'].shift(1))], axis=1).max(axis=1)\n    atr = tr.ewm(span=14, adjust=False).mean()\n    df.loc&#91;:, '+DI'] = 100 * (df&#91;'+DM'].ewm(span=14, adjust=False).mean() \/ atr)\n    df.loc&#91;:, '-DI'] = 100 * (df&#91;'-DM'].ewm(span=14, adjust=False).mean() \/ atr)\n    dx = 100 * np.abs(df&#91;'+DI'] - df&#91;'-DI']) \/ (df&#91;'+DI'] + df&#91;'-DI'])\n    df.loc&#91;:, 'ADX'] = dx.ewm(span=14, adjust=False).mean()\n\n    # Drop NaN values\n    df.dropna(inplace=True)\n\n    # Keep only the required columns\n    df = df&#91;&#91;'Open', 'High', 'Low', 'Close', 'Volume', 'MACD', 'Signal', 'RSI', 'CCI', 'ADX']]\n\n    return df\n\n# -----------------------------------------------------------------------------\n\n# add technical indicators to the training data for each stock\nfor ticker, df in training_data.items():\n    training_data&#91;ticker] = add_technical_indicators(df)\n\n# add technical indicators to the validation data for each stock\nfor ticker, df in validation_data.items():\n    validation_data&#91;ticker] = add_technical_indicators(df)\n\n# add technical indicators to the test data for each stock\nfor ticker, df in test_data.items():\n    test_data&#91;ticker] = add_technical_indicators(df)\n\n# print the first 5 rows of the data\nprint(f'- Training data shape for {ticker}: {training_data&#91;ticker].shape}')\nprint(f'- Validation data shape for {ticker}: {validation_data&#91;ticker].shape}')\nprint(f'- Test data shape for {ticker}: {test_data&#91;ticker].shape}\\n')\n\ndisplay(test_data&#91;ticker].head())<\/code><\/code><\/pre>\n\n\n\n<p>\u76f8\u5e94\u7684\u7ed3\u679c\u5982\u4e0b\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" src=\"https:\/\/www.laoyulaoyu.com\/wp-content\/uploads\/2024\/09\/image-239.png\" alt=\"\" class=\"wp-image-2279\"\/><\/figure>\n\n\n\n<p><\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><strong>4.3 \u81ea\u5b9a\u4e49\u4ea4\u6613\u73af\u5883<\/strong><\/h3>\n\n\n\n<p>\u5728\u8fd9\u91cc\uff0c\u6211\u4eec\u5c06\u4f7f\u7528 <strong>OpenAI Gym \u6846\u67b6<\/strong>\u4e3a\u5f3a\u5316\u5b66\u4e60\u6a21\u578b\u5b9a\u4e49\u4e00\u4e2a\u81ea\u5b9a\u4e49\u4ea4\u6613\u73af\u5883\u3002\u8be5\u73af\u5883\u6a21\u62df\u80a1\u7968\u4ea4\u6613\uff0c\u5141\u8bb8\u4ee3\u7406\u901a\u8fc7\u8d2d\u4e70\u3001\u51fa\u552e\u6216\u6301\u6709\u80a1\u7968\u7b49\u64cd\u4f5c\u4e0e\u5e02\u573a\u4e92\u52a8\u3002<\/p>\n\n\n\n<p id=\"6d69\">\u73af\u5883\u7684\u4e3b\u8981\u7279\u5f81\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u521d\u59cb\u5316\uff1a<\/strong>\u4f7f\u7528\u5386\u53f2\u80a1\u7968\u6570\u636e\u5bf9\u73af\u5883\u8fdb\u884c\u521d\u59cb\u5316\uff0c\u5e76\u8bbe\u7f6e\u5404\u79cd\u53c2\u6570\uff0c\u5305\u62ec\u884c\u52a8\u7a7a\u95f4\u548c\u89c2\u5bdf\u7a7a\u95f4\u3001\u4ea4\u6613\u6210\u672c\u4ee5\u53ca\u8d26\u6237\u53d8\u91cf\uff08\u5982\u4f59\u989d\u3001\u51c0\u8d44\u4ea7\u548c\u6240\u6301\u80a1\u7968\uff09\u3002<\/li>\n\n\n\n<li><strong>\u89c2\u5bdf\u7a7a\u95f4\uff1a<\/strong>\u5728\u6bcf\u4e2a\u6b65\u9aa4\u4e2d\uff0c\u73af\u5883\u90fd\u4f1a\u63d0\u4f9b\u4e00\u4e2a\u7efc\u5408\u72b6\u6001\uff0c\u5176\u4e2d\u5305\u62ec\u5f53\u524d\u80a1\u7968\u4ef7\u683c\u3001\u8d26\u6237\u4f59\u989d\u3001\u6240\u6301\u80a1\u7968\u3001\u51c0\u8d44\u4ea7\u548c\u5176\u4ed6\u76f8\u5173\u6307\u6807\u3002\u8fd9\u4e2a\u89c2\u5bdf\u7a7a\u95f4\u5bf9\u4e8e\u4ee3\u7406\u505a\u51fa\u660e\u667a\u51b3\u7b56\u81f3\u5173\u91cd\u8981\u3002<\/li>\n\n\n\n<li><strong>\u884c\u52a8\u7a7a\u95f4\uff1a<\/strong>\u884c\u52a8\u7a7a\u95f4\u88ab\u5b9a\u4e49\u4e3a\u4e00\u4e2a\u8fde\u7eed\u7684\u7a7a\u95f4\uff0c\u5728\u8fd9\u4e2a\u7a7a\u95f4\u4e2d\uff0c\u4ee3\u7406\u53ef\u4ee5\u51b3\u5b9a\u4e70\u5165\u6216\u5356\u51fa\u6bcf\u53ea\u80a1\u7968\u5728\u6295\u8d44\u7ec4\u5408\u4e2d\u6240\u5360\u7684\u6bd4\u4f8b\u3002\u6b63\u503c\u4ee3\u8868\u4e70\u5165\u884c\u52a8\uff0c\u8d1f\u503c\u4ee3\u8868\u5356\u51fa\u884c\u52a8\u3002<\/li>\n\n\n\n<li><strong>\u6b65\u9aa4\u51fd\u6570\uff1a<\/strong>\u5206\u6b65\u51fd\u6570\u6267\u884c\u4ee3\u7406\u7684\u884c\u52a8\uff0c\u66f4\u65b0\u8d26\u6237\u4f59\u989d\u548c\u6301\u6709\u7684\u80a1\u7968\uff0c\u8ba1\u7b97\u65b0\u7684\u51c0\u503c\uff0c\u5e76\u786e\u5b9a\u5956\u52b1\u3002\u5b83\u8fd8\u8d1f\u8d23\u7ba1\u7406\u4ea4\u6613\u6210\u672c\uff0c\u5e76\u68c0\u67e5\u662f\u5426\u5e94\u6839\u636e\u6700\u5927\u6b65\u6570\u6216\u51c0\u503c\u4f4e\u4e8e\u96f6\u65f6\u7ed3\u675f\u672c\u96c6\u3002<\/li>\n\n\n\n<li><strong>\u6e32\u67d3\uff1a<\/strong>\u6e32\u67d3\u529f\u80fd\u53ef\u63d0\u4f9b\u5f53\u524d\u72b6\u6001\u7684\u4eba\u5de5\u53ef\u8bfb\u8f93\u51fa\uff0c\u5305\u62ec\u6b65\u6570\u3001\u4f59\u989d\u3001\u6240\u6301\u80a1\u4efd\u3001\u51c0\u503c\u548c\u5229\u6da6\u3002<\/li>\n\n\n\n<li><strong>\u91cd\u7f6e\uff1a<\/strong>\u91cd\u7f6e\u529f\u80fd\u53ef\u4e3a\u65b0\u7684\u4e8b\u4ef6\u91cd\u65b0\u521d\u59cb\u5316\u73af\u5883\uff0c\u786e\u4fdd\u4ee3\u7406\u4ee5\u521d\u59cb\u6761\u4ef6\u548c\u6570\u636e\u542f\u52a8\u3002<\/li>\n<\/ul>\n\n\n\n<p id=\"3a85\">\u8fd9\u79cd\u5b9a\u5236\u73af\u5883\u65e8\u5728\u5bc6\u5207\u6a21\u62df\u771f\u5b9e\u4e16\u754c\u7684\u4ea4\u6613\u573a\u666f\uff0c\u4e3a\u5f3a\u5316\u5b66\u4e60\u4ee3\u7406\u63d0\u4f9b\u5b66\u4e60\u548c\u4f18\u5316\u4ea4\u6613\u7b56\u7565\u7684\u5fc5\u8981\u5de5\u5177\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code>class StockTradingEnv(gym.Env):\n\n    metadata = {'render_modes': &#91;'human']}\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def __init__(self, stock_data, transaction_cost_percent=0.005):\n        super(StockTradingEnv, self).__init__()\n        \"\"\"\n        This function initializes the environment with stock data and sets up necessary variables:\n        - Action and Observation Space: Defines the action space (buy\/sell\/hold) and\n                                        observation space (stock prices, balance, shares held, net worth, etc.).\n        - Account Variables: Initializes balance, net worth, shares held, and transaction costs.\n        \"\"\"\n\n        # Remove any empty DataFrames\n        self.stock_data = {ticker: df for ticker, df in stock_data.items() if not df.empty}\n        self.tickers = list(self.stock_data.keys())\n\n        if not self.tickers:\n            raise ValueError(\"All provided stock data is empty\")\n\n        # Calculate the size of one stock's data\n        sample_df = next(iter(self.stock_data.values()))\n        self.n_features = len(sample_df.columns)\n\n        # Define action and observation space\n        self.action_space = spaces.Box(low=-1, high=1, shape=(len(self.tickers),), dtype=np.float32)\n\n        # Observation space: price data for each stock + balance + shares held + net worth + max net worth + current step\n        self.obs_shape = self.n_features * len(self.tickers) + 2 + len(self.tickers) + 2\n        self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape=(self.obs_shape,), dtype=np.float32)\n\n        # Initialize account balance\n        self.initial_balance = 1000\n        self.balance = self.initial_balance\n        self.net_worth = self.initial_balance\n        self.max_net_worth = self.initial_balance\n        self.shares_held = {ticker: 0 for ticker in self.tickers}\n        self.total_shares_sold = {ticker: 0 for ticker in self.tickers}\n        self.total_sales_value = {ticker: 0 for ticker in self.tickers}\n\n        # Set the current step\n        self.current_step = 0\n\n        # Calculate the minimum length of data across all stocks\n        self.max_steps = max(0, min(len(df) for df in self.stock_data.values()) - 1)\n\n        # Transaction cost\n        self.transaction_cost_percent = transaction_cost_percent\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def reset(self, seed=None, options=None):\n        super().reset(seed=seed)\n        \"\"\" Resets the environment to its initial state for a new episode. \"\"\"\n\n        # Reset the account balance\n        self.balance = self.initial_balance\n        self.net_worth = self.initial_balance\n        self.max_net_worth = self.initial_balance\n        self.shares_held = {ticker: 0 for ticker in self.tickers}\n        self.total_shares_sold = {ticker: 0 for ticker in self.tickers}\n        self.total_sales_value = {ticker: 0 for ticker in self.tickers}\n        self.current_step = 0\n        return self._next_observation(), {}\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def _next_observation(self):\n        \"\"\" Returns the current state of the environment, including stock prices, balance, shares held, net worth, etc. \"\"\"\n\n        # initialize the frame\n        frame = np.zeros(self.obs_shape)\n\n        # Add stock data for each ticker\n        idx = 0\n        # Loop through each ticker\n        for ticker in self.tickers:\n            # Get the DataFrame for the current ticker\n            df = self.stock_data&#91;ticker]\n            # If the current step is less than the length of the DataFrame, add the price data for the current step\n            if self.current_step &lt; len(df):\n                frame&#91;idx:idx+self.n_features] = df.iloc&#91;self.current_step].values\n            # Otherwise, add the last price data available\n            elif len(df) &gt; 0:\n                frame&#91;idx:idx+self.n_features] = df.iloc&#91;-1].values\n            # Move the index to the next ticker\n            idx += self.n_features\n\n        # Add balance, shares held, net worth, max net worth, and current step\n        frame&#91;-4-len(self.tickers)] = self.balance # Balance\n        frame&#91;-3-len(self.tickers):-3] = &#91;self.shares_held&#91;ticker] for ticker in self.tickers] # Shares held\n        frame&#91;-3] = self.net_worth # Net worth\n        frame&#91;-2] = self.max_net_worth # Max net worth\n        frame&#91;-1] = self.current_step # Current step\n\n        return frame\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def step(self, actions):\n        \"\"\" Executes an action in the environment, updates the state, calculates rewards, and checks if the episode is done. \"\"\"\n\n        # update the current step\n        self.current_step += 1\n\n        # check if we have reached the maximum number of steps\n        if self.current_step &gt; self.max_steps:\n            return self._next_observation(), 0, True, False, {}\n\n        current_prices = {}\n        # Loop through each ticker and perform the action\n        for i, ticker in enumerate(self.tickers):\n            # Get the current price of the stock\n            current_prices&#91;ticker] = self.stock_data&#91;ticker].iloc&#91;self.current_step]&#91;'Close']\n            # get the action for the current ticker\n            action = actions&#91;i]\n\n            if action &gt; 0:  # Buy\n                # Calculate the number of shares to buy\n                shares_to_buy = int(self.balance * action \/ current_prices&#91;ticker])\n                # Calculate the cost of the shares\n                cost = shares_to_buy * current_prices&#91;ticker]\n                # Transaction cost\n                transaction_cost = cost * self.transaction_cost_percent\n                # Update the balance and shares held\n                self.balance -= (cost + transaction_cost)\n                # Update the total shares sold\n                self.shares_held&#91;ticker] += shares_to_buy\n\n            elif action &lt; 0:  # Sell\n                # Calculate the number of shares to sell\n                shares_to_sell = int(self.shares_held&#91;ticker] * abs(action))\n                # Calculate the sale value\n                sale = shares_to_sell * current_prices&#91;ticker]\n                # Transaction cost\n                transaction_cost = sale * self.transaction_cost_percent\n                # Update the balance and shares held\n                self.balance += (sale - transaction_cost)\n                # Update the total shares sold\n                self.shares_held&#91;ticker] -= shares_to_sell\n                # Update the shares sold\n                self.total_shares_sold&#91;ticker] += shares_to_sell\n                # Update the total sales value\n                self.total_sales_value&#91;ticker] += sale\n\n        # Calculate the net worth\n        self.net_worth = self.balance + sum(self.shares_held&#91;ticker] * current_prices&#91;ticker] for ticker in self.tickers)\n\n        # Update the max net worth\n        self.max_net_worth = max(self.net_worth, self.max_net_worth)\n\n        # Calculate the reward\n        reward = self.net_worth - self.initial_balance\n\n        # Check if the episode is done\n        done = self.net_worth &lt;= 0 or self.current_step &gt;= self.max_steps\n\n        obs = self._next_observation()\n        return obs, reward, done, False, {}\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def render(self, mode='human'):\n        \"\"\" Displays the current state of the environment in a human-readable format. \"\"\"\n\n        # Print the current step, balance, shares held, net worth, and profit\n        profit = self.net_worth - self.initial_balance\n        print(f'Step: {self.current_step}')\n        print(f'Balance: {self.balance:.2f}')\n        for ticker in self.tickers:\n            print(f'{ticker} Shares held: {self.shares_held&#91;ticker]}')\n        print(f'Net worth: {self.net_worth:.2f}')\n        print(f'Profit: {profit:.2f}')\n\n    def close(self):\n        \"\"\" Placeholder for any cleanup operations \"\"\"\n        pass<\/code><\/code><\/pre>\n\n\n\n<p><\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><strong>4.4 \u5efa\u7acb\u5f3a\u5316\u5b66\u4e60\u4ee3\u7406<\/strong><\/h3>\n\n\n\n<p id=\"e499\">\u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u5efa\u7acb\u4e86\u5404\u79cd\u5f3a\u5316\u5b66\u4e60\u4ee3\u7406\uff0c\u4e0e\u524d\u9762\u5b9a\u4e49\u7684\u4ea4\u6613\u73af\u5883\u8fdb\u884c\u4e92\u52a8\u3002\u6bcf\u4e2a\u4ee3\u7406\u90fd\u57fa\u4e8e\u4e0d\u540c\u7684\u5f3a\u5316\u5b66\u4e60\u7b97\u6cd5\uff0c\u800c\u96c6\u5408\u4ee3\u7406\u5219\u7efc\u5408\u4e86\u8fd9\u4e9b\u5355\u4e2a\u6a21\u578b\u7684\u4f18\u52bf\u3002<\/p>\n\n\n\n<h4 class=\"wp-block-heading\"><strong>4.4.1 PolicyGradientLossCallback Class<\/strong><\/h4>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>: \u6b64\u81ea\u5b9a\u4e49\u56de\u8c03\u8bb0\u5f55\u57f9\u8bad\u671f\u95f4\u7684\u7b56\u7565\u68af\u5ea6\u635f\u5931\uff0c\u4ee5\u4fbf\u8fdb\u884c\u6027\u80fd\u76d1\u63a7\u3002<\/li>\n\n\n\n<li><strong>Functionality<\/strong>:\n<ul class=\"wp-block-list\">\n<li><code>_on_step<\/code>: \u4ece\u6a21\u578b\u7684\u65e5\u5fd7\u8bb0\u5f55\u5668\u4e2d\u6355\u6349\u5e76\u9644\u52a0\u7b56\u7565\u68af\u5ea6\u635f\u5931\u3002<\/li>\n\n\n\n<li><code>_on_training_end<\/code>: \u7ed8\u5236\u8bad\u7ec3\u540e\u7684\u7b56\u7565\u68af\u5ea6\u635f\u5931\u56fe\uff0c\u4ee5\u76f4\u89c2\u663e\u793a\u635f\u5931\u968f\u65f6\u95f4\u7684\u53d8\u5316\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\"><strong>4.4.2  Individual Trading Agents<\/strong><\/h4>\n\n\n\n<p><strong>PPOAgent (\u8fd1\u7aef\u7b56\u7565\u4f18\u5316)<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Initialization<\/strong>: \u7528\u6307\u5b9a\u7684\u65f6\u95f4\u6b65\u6570\u548c\u884c\u52a8\u51b3\u7b56\u9608\u503c\u8bbe\u7f6e PPO \u6a21\u578b\u3002<\/li>\n\n\n\n<li><strong>Methods<\/strong>:\n<ul class=\"wp-block-list\">\n<li><code>predict<\/code>: \u8fd4\u56de PPO \u6a21\u578b\u9488\u5bf9\u7ed9\u5b9a\u89c2\u6d4b\u6570\u636e\u6240\u51b3\u5b9a\u7684\u884c\u52a8\u3002<\/li>\n\n\n\n<li>action_to_recommendation\uff08\u884c\u52a8\u5efa\u8bae\uff09\uff1a\u6839\u636e\u9608\u503c\u5c06\u6a21\u578b\u7684\u64cd\u4f5c\u8f6c\u6362\u4e3a\u4ea4\u6613\u5efa\u8bae\uff08\u4e70\u5165\/\u5356\u51fa\/\u6301\u6709\uff09\u3002<\/li>\n\n\n\n<li><code>validate<\/code>: \u901a\u8fc7\u5728\u73af\u5883\u4e2d\u8fd0\u884c\u4ee3\u7406\u5e76\u8ba1\u7b97\u603b\u5956\u52b1\u6765\u8bc4\u4f30\u4ee3\u7406\u7684\u6027\u80fd\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\" id=\"c6fc\"><strong>4.4.3  Other agents<\/strong><\/h4>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>A2CAgent (Advantage Actor-Critic)<\/strong><\/li>\n\n\n\n<li><strong>DDPGAgent (Deep Deterministic Policy Gradient)<\/strong><\/li>\n\n\n\n<li><strong>SACAgent (Soft Actor-Critic)<\/strong><\/li>\n\n\n\n<li><strong>TD3Agent (Twin Delayed Deep Deterministic Policy Gradient)<\/strong><\/li>\n\n\n\n<li><strong>Initialization<\/strong>\uff1a\u521d\u59cb\u5316\u7ee7\u627f\u81ea PPOAgent\uff0c\u4f46\u5206\u522b\u4f7f\u7528 A2C\u3001DDPG\u3001SAC \u548c TD3 \u7b97\u6cd5\u3002\u5b83\u4eec\u8fd8\u5305\u62ec PolicyGradientLossCallback\uff0c\u7528\u4e8e\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u8ddf\u8e2a\u635f\u5931\u3002<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\" id=\"7268\"><strong>4.3.4  EnsembleAgent<\/strong><\/h4>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>: \u7efc\u5408\u591a\u4e2a\u5355\u72ec\u6a21\u578b\uff08PPO\u3001A2C\u3001DDPG\u3001SAC \u548c TD3\uff09\u7684\u9884\u6d4b\u7ed3\u679c\uff0c\u505a\u51fa\u6700\u7ec8\u51b3\u5b9a\u3002\u8fd9\u79cd\u96c6\u5408\u65b9\u6cd5\u65e8\u5728\u5145\u5206\u5229\u7528\u6bcf\u79cd\u7b97\u6cd5\u7684\u4f18\u52bf\u3002<\/li>\n\n\n\n<li><strong>Methods<\/strong>:\n<ul class=\"wp-block-list\">\n<li><strong><code>predict<\/code>: <\/strong>\u5e73\u5747\u6bcf\u4e2a\u5355\u72ec\u6a21\u578b\u9884\u6d4b\u7684\u884c\u52a8\uff0c\u4ee5\u786e\u5b9a\u96c6\u5408\u884c\u52a8\u3002<\/li>\n\n\n\n<li><strong><code>action_to_recommendation<\/code>: <\/strong>\u6839\u636e\u9608\u503c\u5c06\u96c6\u5408\u884c\u52a8\u8f6c\u5316\u4e3a\u4e70\u5165\/\u5356\u51fa\/\u6301\u6709\u5efa\u8bae\u3002<\/li>\n\n\n\n<li><strong><code>validate<\/code>: <\/strong>\u6d4b\u8bd5\u96c6\u5408\u4ee3\u7406\u5728\u73af\u5883\u4e2d\u7684\u8868\u73b0\uff0c\u5e76\u8ba1\u7b97\u603b\u5956\u52b1\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n\n\n\n<p id=\"cfef\">\u8fd9\u4e9b\u4ee3\u7406\u88ab\u8bbe\u8ba1\u7528\u4e8e\u5904\u7406\u73af\u5883\u4e2d\u7684\u4ea4\u6613\u51b3\u7b56\uff0c\u5e76\u7ecf\u8fc7\u9a8c\u8bc1\uff0c\u4ee5\u786e\u4fdd\u5b83\u4eec\u5728\u6700\u5927\u9650\u5ea6\u5730\u63d0\u9ad8\u56de\u62a5\u548c\u505a\u51fa\u660e\u667a\u7684\u4ea4\u6613\u9009\u62e9\u65b9\u9762\u7684\u6709\u6548\u6027\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><strong>4.5 \u7528\u4e8e\u8bad\u7ec3\u548c\u8bc4\u4f30\u4ea4\u6613\u4ee3\u7406\u7684\u8f85\u52a9\u529f\u80fd<\/strong><\/h3>\n\n\n\n<pre class=\"wp-block-code\"><code><code>class PolicyGradientLossCallback(BaseCallback):\n    \"\"\"\n    A custom callback class that logs the policy_gradient_loss during training.\n    This class extends BaseCallback and used to capture and store the metrics we want.\n    \"\"\"\n\n    def __init__(self, verbose=0):\n        super(PolicyGradientLossCallback, self).__init__(verbose)\n        self.losses = &#91;]\n\n    def _on_step(self) -&gt; bool:\n        if hasattr(self.model, 'logger'):\n            logs = self.model.logger.name_to_value\n            if 'train\/policy_gradient_loss' in logs:\n                loss = logs&#91;'train\/policy_gradient_loss']\n                self.losses.append(loss)\n        return True\n\n    def _on_training_end(self):\n        \"\"\" Plot the loss after training ends \"\"\"\n        name = self.model.__class__.__name__\n        plt.figure(figsize=(12, 4))\n        plt.plot(self.losses, label='Policy Gradient Loss')\n        plt.title(f'{name} - Policy Gradient Loss During Training')\n        plt.xlabel('Training Steps')\n        plt.ylabel('Loss')\n        plt.legend()\n        plt.show()<\/code><\/code><\/pre>\n\n\n\n<pre class=\"wp-block-code\"><code><code># Define PPO Agent\nclass PPOAgent:\n\n    def __init__(self, env, total_timesteps, threshold):\n        self.model = PPO(\"MlpPolicy\", env, verbose=1)\n        self.callback = PolicyGradientLossCallback()\n        self.model.learn(total_timesteps=total_timesteps, callback=self.callback)\n        self.threshold = threshold\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def predict(self, obs):\n        action, _ = self.model.predict(obs, deterministic=True)\n        return action\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def action_to_recommendation(self, action):\n        recommendations = &#91;]\n        for a in action:\n            if a &gt; self.threshold:\n                recommendations.append('buy')\n            elif a &lt; -self.threshold:\n                recommendations.append('sell')\n            else:\n                recommendations.append('hold')\n        return recommendations\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def validate(self, env):\n        obs = env.reset()\n        total_rewards = 0\n        for _ in range(1000):  # Adjust based on needs\n            action, _ = self.model.predict(obs)\n            obs, reward, done, _ = env.step(action)\n            total_rewards += reward\n            if done:\n                obs = env.reset()\n        print(f'Agent Validation Reward: {total_rewards}')\n\n# -----------------------------------------------------------------------------\n\n# Define A2C Agent\nclass A2CAgent(PPOAgent):\n    def __init__(self, env, total_timesteps, threshold):\n        super().__init__(env, total_timesteps, threshold)\n        self.model = A2C(\"MlpPolicy\", env, verbose=1)\n        self.callback = PolicyGradientLossCallback()\n        self.model.learn(total_timesteps=total_timesteps, callback=self.callback)\n\n# -----------------------------------------------------------------------------\n\n# Define DDPG Agent\nclass DDPGAgent(PPOAgent):\n    def __init__(self, env, total_timesteps, threshold):\n        super().__init__(env, total_timesteps, threshold)\n        self.model = DDPG(\"MlpPolicy\", env, verbose=1)\n        self.callback = PolicyGradientLossCallback()\n        self.model.learn(total_timesteps=total_timesteps, callback=self.callback)\n\n# -----------------------------------------------------------------------------\n\n# Define SAC Agent\nclass SACAgent(PPOAgent):\n    def __init__(self, env, total_timesteps, threshold):\n        super().__init__(env, total_timesteps, threshold)\n        self.model = SAC(\"MlpPolicy\", env, verbose=1)\n        self.callback = PolicyGradientLossCallback()\n        self.model.learn(total_timesteps=total_timesteps, callback=self.callback)\n\n# -----------------------------------------------------------------------------\n\n# Define TD3 Agent\nclass TD3Agent(PPOAgent):\n    def __init__(self, env, total_timesteps, threshold):\n        super().__init__(env, total_timesteps, threshold)\n        self.model = TD3(\"MlpPolicy\", env, verbose=1)\n        self.callback = PolicyGradientLossCallback()\n        self.model.learn(total_timesteps=total_timesteps, callback=self.callback)\n\n# -----------------------------------------------------------------------------\n\n# Define Ensemble Agent\nclass EnsembleAgent:\n\n    def __init__(self, ppo_model, a2c_model, ddpg_model, sac_model, td3_model, threshold):\n        self.ppo_model = ppo_model\n        self.a2c_model = a2c_model\n        self.ddpg_model = ddpg_model\n        self.sac_model = sac_model\n        self.td3_model = td3_model\n        self.threshold = threshold\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def predict(self, obs):\n        ppo_action, _ = self.ppo_model.predict(obs, deterministic=True)\n        a2c_action, _ = self.a2c_model.predict(obs, deterministic=True)\n        ddpg_action, _ = self.ddpg_model.predict(obs, deterministic=True)\n        sac_action, _ = self.sac_model.predict(obs, deterministic=True)\n        td3_action, _ = self.td3_model.predict(obs, deterministic=True)\n\n        # Average the actions\n        ensemble_action = np.mean(&#91;ppo_action, a2c_action, ddpg_action, sac_action, td3_action], axis=0)\n        return ensemble_action\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def action_to_recommendation(self, action):\n        recommendations = &#91;]\n        for a in action:\n            if a &gt; self.threshold:\n                recommendations.append('buy')\n            elif a &lt; -self.threshold:\n                recommendations.append('sell')\n            else:\n                recommendations.append('hold')\n        return recommendations\n\n    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n\n    def validate(self, env):\n        obs = env.reset()\n        total_rewards = 0\n        for _ in range(1000):  # Adjust based on needs\n            action = self.predict(obs)\n            obs, reward, done, _ = env.step(action)\n            total_rewards += reward\n            if done:\n                obs = env.reset()\n        print(f'Agent Validation Reward: {total_rewards}')<\/code><\/code><\/pre>\n\n\n\n<p>\u4ee5\u4e0b\u662f\u7528\u4e8e\u8bad\u7ec3\u548c\u8bc4\u4f30\u4ea4\u6613\u4ee3\u7406\u7684\u8f85\u52a9\u529f\u80fd\u7684\u8be6\u7ec6\u6982\u8ff0\uff1a<\/p>\n\n\n\n<h4 class=\"wp-block-heading\"><strong>4.5.1 \u521b\u9020\u73af\u5883\u548c\u57f9\u8bad\u4ee3\u7406\u7684\u529f\u80fd<\/strong><\/h4>\n\n\n\n<p><strong><code>create_env_and_train_agents<\/code><\/strong> <strong>\uff08\u521b\u5efa\u73af\u5883\u548c\u57f9\u8bad\u4ee3\u7406\uff09<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>:\u521d\u59cb\u5316\u4ea4\u6613\u73af\u5883\u5e76\u57f9\u8bad\u5404\u79cd\u4ea4\u6613\u4ee3\u7406\u3002<\/li>\n\n\n\n<li><strong>Functionality\uff08\u529f\u80fd\u6027\uff09:<\/strong><\/li>\n\n\n\n<li><strong>Environments<\/strong>:\u4f7f\u7528 StockTradingEnv \u7c7b\u521b\u5efa\u8bad\u7ec3 (train_env) \u548c\u9a8c\u8bc1 (val_env) \u73af\u5883\u3002<\/li>\n\n\n\n<li><strong>Agents<\/strong>: \u4f7f\u7528\u5404\u81ea\u7684\u7c7b\u522b\u548c\u9a8c\u8bc1\u6570\u636e\u5bf9\u6bcf\u4e2a\u4ee3\u7406\uff08PPO\u3001A2C\u3001DDPG\u3001SAC\u3001TD3\uff09\u8fdb\u884c\u57f9\u8bad\u548c\u9a8c\u8bc1\u3002<\/li>\n\n\n\n<li><strong>Ensemble<\/strong>: \u8bad\u7ec3\u5e76\u9a8c\u8bc1\u4e00\u4e2a\u96c6\u5408\u4ee3\u7406\uff0c\u8be5\u4ee3\u7406\u7efc\u5408\u4e86\u6240\u6709\u5355\u4e2a\u6a21\u578b\u7684\u9884\u6d4b\u7ed3\u679c\u3002<\/li>\n\n\n\n<li><strong>Returns<\/strong>: \u63d0\u4f9b\u521d\u59cb\u5316\u73af\u5883\u548c\u8bad\u7ec3\u6709\u7d20\u7684\u4ee3\u7406\uff0c\u4f9b\u8fdb\u4e00\u6b65\u5206\u6790\u3002<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\"><strong>4.5.2 \u53ef\u89c6\u5316\u529f\u80fd<\/strong><\/h4>\n\n\n\n<p id=\"19f3\"><strong><code>visualize_portfolio<\/code> \uff08\u53ef\u89c6\u5316\u7ec4\u5408\uff09<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>: \u663e\u793a\u4f59\u989d\u3001\u51c0\u503c\u548c\u6240\u6301\u80a1\u7968\u7684\u65f6\u95f4\u53d8\u5316\u3002<\/li>\n\n\n\n<li><strong>Parameters<\/strong>\uff08<strong>\u53c2\u6570\uff09:<\/strong>\n<ul class=\"wp-block-list\">\n<li><code>steps<\/code>:\u65f6\u95f4\u6b65\u9aa4\u5217\u8868\u3002<\/li>\n\n\n\n<li><code>balances<\/code>,&nbsp;<code>net_worths<\/code>,&nbsp;<code>shares_held<\/code>: \u957f\u671f\u8ddf\u8e2a\u7684\u6307\u6807\u3002<\/li>\n\n\n\n<li><code>tickers<\/code>: \u80a1\u7968\u4ee3\u7801\u5217\u8868\u3002<\/li>\n\n\n\n<li><code>show_balance<\/code>,&nbsp;<code>show_net_worth<\/code>,&nbsp;<code>show_shares_held<\/code>:\u663e\u793a\u5e73\u8861\uff08show_balance\uff09\u3001\u663e\u793a\u51c0\u503c\uff08show_net_worth\uff09\u3001\u663e\u793a\u6240\u6301\u80a1\u7968\uff08show_shares_held\uff09\uff1a\u7528\u4e8e\u63a7\u5236\u663e\u793a\u54ea\u4e9b\u56fe\u8868\u7684\u6807\u5fd7\u3002<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li><strong>Functionality<\/strong>: \u529f\u80fd\u4e3a\u4f59\u989d\u3001\u51c0\u503c\u548c\u6240\u6301\u80a1\u7968\u521b\u5efa\u591a\u9762\u677f\u56fe\uff0c\u53ef\u76f4\u89c2\u67e5\u770b\u6295\u8d44\u7ec4\u5408\u7684\u957f\u671f\u8868\u73b0\u3002<\/li>\n<\/ul>\n\n\n\n<p id=\"f7f2\"><code><strong>visualize_portfolio_net_worth<\/strong><\/code><strong> \uff08\u53ef\u89c6\u5316\u6295\u8d44\u7ec4\u5408\u51c0\u503c\uff09<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>: \u7ed8\u5236\u4e00\u6bb5\u65f6\u95f4\u5185\u7684\u51c0\u8d44\u4ea7\u56fe\u3002<\/li>\n\n\n\n<li><strong>Parameters\uff08\u53c2\u6570\uff09:<\/strong>\n<ul class=\"wp-block-list\">\n<li><code>steps<\/code>: \u65f6\u95f4\u6b65\u9aa4\u5217\u8868\u3002<\/li>\n\n\n\n<li><code>net_worths<\/code>: \u957f\u671f\u8ddf\u8e2a\u7684\u51c0\u8d44\u4ea7\u3002<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li><strong>Functionality<\/strong>:\u521b\u5efa\u5355\u4e00\u7684\u51c0\u503c\u56fe\uff0c\u63d0\u4f9b\u6295\u8d44\u7ec4\u5408\u4ef7\u503c\u589e\u957f\u7684\u6e05\u6670\u89c6\u56fe\u3002<\/li>\n<\/ul>\n\n\n\n<p id=\"9a31\"><code><strong>visualize_multiple_portfolio_net_worth<\/strong><\/code> <strong>\uff08\u591a\u4e2a\u53ef\u89c6\u5316\u6295\u8d44\u7ec4\u5408\u51c0\u503c\uff09<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>: \u5728\u540c\u4e00\u56fe\u8868\u4e0a\u6bd4\u8f83\u591a\u4e2a\u6295\u8d44\u7ec4\u5408\u7684\u51c0\u503c\u3002<\/li>\n\n\n\n<li><strong>Parameters\uff08\u53c2\u6570\uff09:<\/strong>\n<ul class=\"wp-block-list\">\n<li><code>steps<\/code>: \u65f6\u95f4\u6b65\u9aa4\u5217\u8868\u3002<\/li>\n\n\n\n<li><code>net_worths_list<\/code>: \u4e0d\u540c\u4ee3\u7406\u5546\u7684\u51c0\u8d44\u4ea7\u7cfb\u5217\u5217\u8868\u3002<\/li>\n\n\n\n<li><code>labels<\/code>:\u6bcf\u4e2a\u4ee3\u7406\u5546\u51c0\u8d44\u4ea7\u7cfb\u5217\u7684\u6807\u7b7e\u3002<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li><strong>Functionality<\/strong>: \u5c06\u591a\u4e2a\u4ee3\u7406\u7684\u51c0\u8d44\u4ea7\u7ed8\u5236\u5728\u4e00\u5f20\u56fe\u8868\u4e0a\uff0c\u4fbf\u4e8e\u76f4\u63a5\u6bd4\u8f83\u3002<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\"><strong>4.5.3 \u6d4b\u8bd5\u529f\u80fd<\/strong><\/h4>\n\n\n\n<p id=\"9438\"><code><strong>test_agent<\/strong><\/code><strong>\uff08\u6d4b\u8bd5\u4ee3\u7406\uff09<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>: \u6d4b\u8bd5\u5355\u4e2a\u4ee3\u7406\u5728\u73af\u5883\u4e2d\u7684\u6027\u80fd\u5e76\u8ddf\u8e2a\u5173\u952e\u6307\u6807\u3002<\/li>\n\n\n\n<li><strong>Parameters\uff08\u53c2\u6570\uff09:<\/strong>\n<ul class=\"wp-block-list\">\n<li><code>env<\/code>:\u6d4b\u8bd5\u4ee3\u7406\u7684\u73af\u5883\u3002<\/li>\n\n\n\n<li><code>agent<\/code>: \u5f85\u6d4b\u4ee3\u7406\u3002<\/li>\n\n\n\n<li><code>stock_data<\/code>: \u7528\u4e8e\u6307\u6807\u8ddf\u8e2a\u7684\u80a1\u7968\u76f8\u5173\u6570\u636e\u3002<\/li>\n\n\n\n<li><code>n_tests<\/code>: \u6d4b\u8bd5\u8fed\u4ee3\u6b21\u6570\u3002<\/li>\n\n\n\n<li><code>visualize<\/code>: \u53ef\u89c6\u5316\uff1a\u6807\u5fd7\uff0c\u7528\u4e8e\u63a7\u5236\u6d4b\u8bd5\u671f\u95f4\u7684\u73af\u5883\u6e32\u67d3\u3002<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li><strong>Functionality<\/strong>: \u5728\u73af\u5883\u4e2d\u8fd0\u884c\u4ee3\u7406\uff0c\u6536\u96c6\u6307\u6807\uff08\u4f59\u989d\u3001\u51c0\u503c\u3001\u6240\u6301\u80a1\u7968\uff09\uff0c\u5e76\u53ef\u9009\u62e9\u5c06\u73af\u5883\u53ef\u89c6\u5316\u3002<\/li>\n<\/ul>\n\n\n\n<p id=\"87dc\"><code><strong>test_and_visualize_agents<\/strong><\/code> <strong> \uff08\u6d4b\u8bd5\u548c\u53ef\u89c6\u5316\u4ee3\u7406\uff09<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>: \u6d4b\u8bd5\u591a\u4e2a\u4ee3\u7406\u5e76\u53ef\u89c6\u5316\u5176\u6027\u80fd\u3002<\/li>\n\n\n\n<li><strong>Parameters\uff08\u53c2\u6570\uff09:<\/strong>\n<ul class=\"wp-block-list\">\n<li><code>env<\/code>: \u6d4b\u8bd5\u4ee3\u7406\u7684\u73af\u5883\u3002<\/li>\n\n\n\n<li><code>agents<\/code>: \u5f85\u6d4b\u5236\u5242\u8bcd\u5178\u3002<\/li>\n\n\n\n<li><code>data<\/code>: \u7528\u4e8e\u6307\u6807\u8ddf\u8e2a\u7684\u5e93\u5b58\u6570\u636e\u3002<\/li>\n\n\n\n<li><code>n_tests<\/code>: \u6d4b\u8bd5\u8fed\u4ee3\u6b21\u6570\u3002<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li><strong>Functionality<\/strong>: \u6d4b\u8bd5\u6bcf\u4e2a\u4ee3\u7406\uff0c\u6536\u96c6\u6027\u80fd\u6307\u6807\uff0c\u5e76\u751f\u6210\u51c0\u503c\u968f\u65f6\u95f4\u53d8\u5316\u7684\u53ef\u89c6\u5316\u5bf9\u6bd4\u56fe\u3002<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\"><br><strong>4.5.4 \u6027\u80fd\u6bd4\u8f83\u529f\u80fd<\/strong><\/h4>\n\n\n\n<p id=\"a5d7\"><code><strong>compare_and_plot_agents<\/strong><\/code><strong>\uff08\u6bd4\u8f83\u548c\u7ed8\u56fe\u4ee3\u7406)<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Purpose<\/strong>: \u6839\u636e\u4ee3\u7406\u7684\u6536\u76ca\u3001\u6807\u51c6\u5dee\u548c\u590f\u666e\u6bd4\u7387\u5bf9\u5176\u8fdb\u884c\u6bd4\u8f83\u3002<\/li>\n\n\n\n<li><strong>Parameters(\u53c2\u6570):<\/strong>\n<ul class=\"wp-block-list\">\n<li><code>agents_metrics<\/code>: \u4ece\u6d4b\u8bd5\u4ee3\u7406\u6536\u96c6\u7684\u6307\u6807\u3002<\/li>\n\n\n\n<li><code>labels<\/code>: \u6bcf\u4e2a\u4ee3\u7406\u7684\u6807\u7b7e\u3002<\/li>\n\n\n\n<li><code>risk_free_rate<\/code>:\u7528\u4e8e\u8ba1\u7b97\u590f\u666e\u6bd4\u7387\u7684\u65e0\u98ce\u9669\u5229\u7387\u3002<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li><strong>Functionality(\u529f\u80fd\u6027):<\/strong>\n<ul class=\"wp-block-list\">\n<li><strong>Comparison<\/strong>: \u8ba1\u7b97\u6bcf\u4e2a\u4ee3\u7406\u7684\u6536\u76ca\u3001\u6807\u51c6\u5dee\u548c\u590f\u666e\u6bd4\u7387\u3002<\/li>\n\n\n\n<li><strong>Visualization<\/strong>: \u663e\u793a\u5206\u7c7b\u6570\u636e\u6846\u548c\u6761\u5f62\u56fe\uff0c\u6bd4\u8f83\u4ee3\u7406\u7684\u590f\u666e\u6bd4\u7387\uff0c\u7a81\u51fa\u663e\u793a\u54ea\u4e2a\u4ee3\u7406\u5728\u98ce\u9669\u8c03\u6574\u540e\u6536\u76ca\u65b9\u9762\u8868\u73b0\u6700\u4f73\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n\n\n\n<p id=\"9397\">\u8fd9\u4e9b\u529f\u80fd\u4e3a\u57f9\u8bad\u3001\u6d4b\u8bd5\u548c\u8bc4\u4f30\u4ea4\u6613\u4ee3\u7406\u63d0\u4f9b\u4e86\u5168\u9762\u7684\u5de5\u5177\u5305\uff0c\u53ef\u5bf9\u4e0d\u540c\u7684\u6a21\u578b\u8fdb\u884c\u6df1\u5165\u5206\u6790\u548c\u6bd4\u8f83\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code># Function to create the environment and train the agents\ndef create_env_and_train_agents(train_data, val_data, total_timesteps, threshold):\n\n    # Create environments for training and validation\n    train_env = DummyVecEnv(&#91;lambda: StockTradingEnv(train_data)])\n    val_env = DummyVecEnv(&#91;lambda: StockTradingEnv(val_data)])\n\n    # Train and Validate PPO Agent\n    ppo_agent = PPOAgent(train_env, total_timesteps, threshold)\n    ppo_agent.validate(val_env)\n\n    # Train and Validate A2C Agent\n    a2c_agent = A2CAgent(train_env, total_timesteps, threshold)\n    a2c_agent.validate(val_env)\n\n    # Train and Validate DDPG Agent\n    ddpg_agent = DDPGAgent(train_env, total_timesteps, threshold)\n    ddpg_agent.validate(val_env)\n\n    # Train and Validate SAC Agent\n    sac_agent = SACAgent(train_env, total_timesteps, threshold)\n    sac_agent.validate(val_env)\n\n    # Train and Validate TD3 Agent\n    td3_agent = TD3Agent(train_env, total_timesteps, threshold)\n    td3_agent.validate(val_env)\n\n    # Train and Validate the ensemble agent\n    ensemble_agent = EnsembleAgent(ppo_agent.model, a2c_agent.model, ddpg_agent.model,\n                                   sac_agent.model, td3_agent.model, threshold)\n    ensemble_agent.validate(val_env)\n\n    return train_env, val_env, ppo_agent, a2c_agent, ddpg_agent, sac_agent, td3_agent, ensemble_agent\n\n# -----------------------------------------------------------------------------\n\n# Function to visualize portfolio changes\ndef visualize_portfolio(steps, balances, net_worths, shares_held, tickers,\n                        show_balance=True, show_net_worth=True, show_shares_held=True):\n\n    fig, axs = plt.subplots(3, figsize=(12, 18))\n\n    # Plot the balance\n    if show_balance:\n        axs&#91;0].plot(steps, balances, label='Balance')\n        axs&#91;0].set_title('Balance Over Time')\n        axs&#91;0].set_xlabel('Steps')\n        axs&#91;0].set_ylabel('Balance')\n        axs&#91;0].legend()\n\n    # Plot the net worth\n    if show_net_worth:\n        axs&#91;1].plot(steps, net_worths, label='Net Worth', color='orange')\n        axs&#91;1].set_title('Net Worth Over Time')\n        axs&#91;1].set_xlabel('Steps')\n        axs&#91;1].set_ylabel('Net Worth')\n        axs&#91;1].legend()\n\n    # Plot the shares held\n    if show_shares_held:\n        for ticker in tickers:\n            axs&#91;2].plot(steps, shares_held&#91;ticker], label=f'Shares Held: {ticker}')\n        axs&#91;2].set_title('Shares Held Over Time')\n        axs&#91;2].set_xlabel('Steps')\n        axs&#91;2].set_ylabel('Shares Held')\n        axs&#91;2].legend()\n\n    plt.tight_layout()\n    plt.show()\n\n# -----------------------------------------------------------------------------\n\n# function to visualize the portfolio net worth\ndef visualize_portfolio_net_worth(steps, net_worths):\n\n    plt.figure(figsize=(12, 6))\n    plt.plot(steps, net_worths, label='Net Worth', color='orange')\n    plt.title('Net Worth Over Time')\n    plt.xlabel('Steps')\n    plt.ylabel('Net Worth')\n    plt.legend()\n    plt.show()\n\n# -----------------------------------------------------------------------------\n\n# function to visualize the multiple portfolio net worths ( same chart )\ndef visualize_multiple_portfolio_net_worth(steps, net_worths_list, labels):\n\n    plt.figure(figsize=(12, 6))\n    for i, net_worths in enumerate(net_worths_list):\n        plt.plot(steps, net_worths, label=labels&#91;i])\n    plt.title('Net Worth Over Time')\n    plt.xlabel('Steps')\n    plt.ylabel('Net Worth')\n    plt.legend()\n    plt.show()\n\n# -----------------------------------------------------------------------------\n\ndef test_agent(env, agent, stock_data, n_tests=1000, visualize=False):\n    \"\"\" Test a single agent and track performance metrics, with an option to visualize the results \"\"\"\n\n    # Initialize metrics tracking\n    metrics = {\n        'steps': &#91;],\n        'balances': &#91;],\n        'net_worths': &#91;],\n        'shares_held': {ticker: &#91;] for ticker in stock_data.keys()}\n    }\n\n    # Reset the environment before starting the tests\n    obs = env.reset()\n\n    for i in range(n_tests):\n\n        metrics&#91;'steps'].append(i)\n\n        action = agent.predict(obs)\n\n        obs, rewards, dones, infos = env.step(action)\n\n        if visualize:\n            env.render()\n\n        # Track metrics\n        metrics&#91;'balances'].append(env.get_attr('balance')&#91;0])\n        metrics&#91;'net_worths'].append(env.get_attr('net_worth')&#91;0])\n        env_shares_held = env.get_attr('shares_held')&#91;0]\n\n        # Update shares held for each ticker\n        for ticker in stock_data.keys():\n            if ticker in env_shares_held:\n                metrics&#91;'shares_held']&#91;ticker].append(env_shares_held&#91;ticker])\n            else:\n                metrics&#91;'shares_held']&#91;ticker].append(0)  # Append 0 if ticker is not found\n\n        if dones:\n            obs = env.reset()\n\n    return metrics\n\n# -----------------------------------------------------------------------------\n\ndef test_and_visualize_agents(env, agents, data, n_tests=1000):\n\n    metrics = {}\n    for agent_name, agent in agents.items():\n        print(f\"Testing {agent_name}...\")\n        metrics&#91;agent_name] = test_agent(env, agent, data, n_tests=n_tests, visualize=True)\n\n    # Extract net worths for visualization\n    net_worths = &#91;metrics&#91;agent_name]&#91;'net_worths'] for agent_name in agents.keys()]\n    steps = next(iter(metrics.values()))&#91;'steps']  # Assuming all agents have the same step count for simplicity\n\n    # Visualize the performance metrics of multiple agents\n    visualize_multiple_portfolio_net_worth(steps, net_worths, list(agents.keys()))\n\n# -----------------------------------------------------------------------------\n\ndef compare_and_plot_agents(agents_metrics, labels, risk_free_rate=0.0):\n\n    # Function to compare returns, standard deviation, and sharpe ratio of agents\n    def compare_agents(agents_metrics, labels):\n        returns = &#91;]\n        stds = &#91;]\n        sharpe_ratios = &#91;]\n\n        for metrics in agents_metrics:\n\n            net_worths = metrics&#91;'net_worths']\n\n            # Calculate daily returns\n            daily_returns = np.diff(net_worths) \/ net_worths&#91;:-1]\n            avg_return = np.mean(daily_returns)\n            std_return = np.std(daily_returns)\n            sharpe_ratio = ((avg_return - risk_free_rate) \/ std_return) if std_return != 0 else 'Inf'\n\n            returns.append(avg_return)\n            stds.append(std_return)\n            sharpe_ratios.append(sharpe_ratio)\n\n        df = pd.DataFrame({\n            'Agent': labels,\n            'Return': returns,\n            'Standard Deviation': stds,\n            'Sharpe Ratio': sharpe_ratios\n        })\n\n        return df\n\n    # Compare agents\n    df = compare_agents(agents_metrics, labels)\n\n    # Sort the dataframe by sharpe ratio\n    df_sorted = df.sort_values(by='Sharpe Ratio', ascending=False)\n\n    # Display the dataframe\n    display(df_sorted)\n\n    # Plot bar chart for sharpe ratio\n    plt.figure(figsize=(12, 6))\n    plt.bar(df_sorted&#91;'Agent'], df_sorted&#91;'Sharpe Ratio'])\n    plt.title('Sharpe Ratio Comparison')\n    plt.xlabel('Agent')\n    plt.ylabel('Sharpe Ratio')\n    plt.show()<\/code><\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><strong>4.6 \u8bad\u7ec3\u4ea4\u6613\u4ee3\u7406<\/strong><\/h3>\n\n\n\n<h4 class=\"wp-block-heading\" id=\"5b2e\"><br><strong>4.6.1 \u8bad\u7ec3\u53c2\u6570\u8bbe\u7f6e<\/strong><\/h4>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Threshold<\/strong>: \u9608\u503c\u51b3\u5b9a\u89e6\u53d1\u4e70\u5165\u6216\u5356\u51fa\u51b3\u7b56\u7684\u6700\u5c0f\u52a8\u4f5c\u5e45\u5ea6\u3002\u5728\u672c\u4f8b\u4e2d\uff0c\u9608\u503c\u8bbe\u7f6e\u4e3a 0.1\u3002<\/li>\n\n\n\n<li><strong>Total Timesteps<\/strong>: \u8be5\u53c2\u6570\u6307\u5b9a\u4e86\u8bad\u7ec3\u4ee3\u7406\u7684\u603b\u65f6\u95f4\u6b65\u6570\u3002\u8fd9\u91cc\u8bbe\u7f6e\u4e3a 10,000 \u4e2a\u65f6\u95f4\u6b65\u3002<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\" id=\"7305\"><strong>4.6.2 \u73af\u5883\u521b\u5efa\u548c\u4ee3\u7406\u57f9\u8bad<\/strong><\/h4>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Environment Creation<\/strong>: \u8be5\u6b65\u9aa4\u4f7f\u7528&nbsp;<code>StockTradingEnv<\/code>\u7c7b\u521d\u59cb\u5316\u8bad\u7ec3\u548c\u9a8c\u8bc1\u73af\u5883\uff0c\u5e76\u6839\u636e\u63d0\u4f9b\u7684\u80a1\u7968\u6570\u636e\u8fdb\u884c\u8c03\u6574\u3002<\/li>\n\n\n\n<li><strong>Agent Training<\/strong>: <code>create_env_and_train_agents<\/code>\u51fd\u6570\u4f7f\u7528\u8bad\u7ec3\u73af\u5883\u8bad\u7ec3\u5404\u79cd\u5f3a\u5316\u5b66\u4e60\u4ee3\u7406\uff08PPO\u3001A2C\u3001DDPG\u3001SAC\u3001TD3\uff09\u3002\u6bcf\u4e2a\u4ee3\u7406\u7684\u8bad\u7ec3\u65f6\u95f4\u4e3a\u6307\u5b9a\u7684\u65f6\u95f4\u6b65\u6570\u3002<\/li>\n\n\n\n<li><strong>Ensemble Agent<\/strong>:\u8fd8\u53ef\u4ee5\u8bad\u7ec3\u4e00\u4e2a\u96c6\u5408\u4ee3\u7406\uff0c\u5b83\u7efc\u5408\u4e86\u6240\u6709\u5355\u4e2a\u6a21\u578b\u7684\u9884\u6d4b\u7ed3\u679c\u3002\u8fd9\u79cd\u65b9\u6cd5\u65e8\u5728\u5145\u5206\u5229\u7528\u6bcf\u4e2a\u6a21\u578b\u7684\u4f18\u52bf\uff0c\u5e76\u6709\u53ef\u80fd\u63d0\u9ad8\u6574\u4f53\u6027\u80fd\u3002<\/li>\n<\/ul>\n\n\n\n<p id=\"06c8\">\u8fd4\u56de\u7684\u5bf9\u8c61\u5305\u62ec\u7ecf\u8fc7\u8bad\u7ec3\u7684\u73af\u5883\u548c\u4ee3\u7406\uff0c\u8fd9\u4e9b\u73af\u5883\u548c\u4ee3\u7406\u53ef\u7528\u4e8e\u8fdb\u4e00\u6b65\u7684\u8bc4\u4f30\u548c\u6027\u80fd\u5206\u6790\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code># Create the environment and train the agents\nthreshold = 0.1\ntotal_timesteps = 10000\ntrain_env, val_env, ppo_agent, a2c_agent, ddpg_agent, sac_agent, td3_agent, ensemble_agent = \\\n  create_env_and_train_agents(training_data, validation_data, total_timesteps, threshold)<\/code><\/code><\/pre>\n\n\n\n<p>\u6211\u4eec\u8fd8\u53ef\u4ee5\u5bf9\u4ee3\u7406\u8fdb\u884c\u6d4b\u8bd5\u548c\u53ef\u89c6\u5316\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code>n_tests = 1000\nagents = {\n    'PPO Agent': ppo_agent,\n    'A2C Agent': a2c_agent,\n    'DDPG Agent': ddpg_agent,\n    'SAC Agent': sac_agent,\n    'TD3 Agent': td3_agent,\n    'Ensemble Agent': ensemble_agent\n}\n\ntest_and_visualize_agents(train_env, agents, training_data, n_tests=n_tests)\n\ntest_env = DummyVecEnv(&#91;lambda: StockTradingEnv(test_data)])\ntest_and_visualize_agents(test_env, agents, test_data, n_tests=n_tests)<\/code><\/code><\/pre>\n\n\n\n<p id=\"0dfa\">\u76f8\u5e94\u7684\u7ed3\u679c\u5982\u4e0b\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" src=\"https:\/\/www.laoyulaoyu.com\/wp-content\/uploads\/2024\/09\/image-240.png\" alt=\"\" class=\"wp-image-2283\"\/><\/figure>\n\n\n\n<p>\u6211\u4eec\u8fd8\u6bd4\u8f83\u4e86\u4ee3\u7406\u5728\u6d4b\u8bd5\u6570\u636e\u4e0a\u7684\u8868\u73b0\uff08\u6536\u76ca\u3001\u6807\u51c6\u504f\u5dee\u548c\u9510\u5229\u6bd4\u7387\uff09\u3002<\/p>\n\n\n\n<p>\u4ee3\u7406\u7684\u590f\u666e\u6bd4\u7387\u8d8a\u9ad8\uff0c\u5176\u6536\u76ca\u76f8\u5bf9\u4e8e\u6240\u627f\u62c5\u7684\u6295\u8d44\u98ce\u9669\u800c\u8a00\u5c31\u8d8a\u597d\u3002\u56e0\u6b64\uff0c\u6211\u4eec\u8981\u9009\u62e9\u80fd\u591f\u6700\u5927\u5316\u6536\u76ca\u7684\u4ea4\u6613\u4ee3\u7406\uff0c\u4ee5\u9002\u5e94\u4e0d\u65ad\u589e\u52a0\u7684\u98ce\u9669\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code>test_agents_metrics = &#91;test_agent(test_env, agent, test_data, n_tests=n_tests, visualize=False) for agent in agents.values()]\ncompare_and_plot_agents(test_agents_metrics, list(agents.keys()))<\/code><\/code><\/pre>\n\n\n\n<p>\u76f8\u5e94\u7684\u7ed3\u679c\u5982\u4e0b\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" src=\"https:\/\/www.laoyulaoyu.com\/wp-content\/uploads\/2024\/09\/image-241.png\" alt=\"\" class=\"wp-image-2284\"\/><\/figure>\n\n\n\n<p>\u6700\u540e\uff0c\u6211\u4eec\u8fd8\u53ef\u4ee5\u5229\u7528\u8be5\u6a21\u578b\u63d0\u51fa\u6b21\u65e5\u5efa\u8bae\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><code>def prepare_next_day_data(stock_data):\n    \"\"\" Prepares the observation for the next trading day \"\"\"\n\n    # Initialize the environment with the current stock data\n    env = StockTradingEnv(stock_data)\n    env.reset()\n\n    # Prepare the next day's observation\n    next_day_observations = env._next_observation()\n\n    return next_day_observations\n\n# -----------------------------------------------------------------------------\n\ndef generate_next_day_recommendations(agents, next_day_observation):\n    \"\"\" Generate recommendations for the next trading day using the trained agents \"\"\"\n\n    recommendations = {agent_name: &#91;] for agent_name in agents.keys()}\n\n    for agent_name, agent in agents.items():\n        action = agent.predict(next_day_observation)\n        recs = agent.action_to_recommendation(action)\n        recommendations&#91;agent_name] = zip(recs, action)\n\n    return recommendations\n\n# -----------------------------------------------------------------------------\n\n# Prepare next day's observation\nnext_day_observation = prepare_next_day_data(test_data)\n\n# Generate recommendations for the next trading day\nrecommendations = generate_next_day_recommendations(agents, next_day_observation)\n\n# Print or display recommendations\nfor agent_name, recs in recommendations.items():\n  if agent_name == 'Ensemble Agent':\n      print(f'\\nRecommendations for {agent_name}:')\n      for ticker, recommendation in zip(tickers, recs):\n          print(f\"{ticker}: {recommendation}\")<\/code><\/code><\/pre>\n\n\n\n<p id=\"d197\">\u76f8\u5e94\u7684\u7ed3\u679c\u5982\u4e0b\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" src=\"https:\/\/www.laoyulaoyu.com\/wp-content\/uploads\/2024\/09\/image-242-613x1024.png\" alt=\"\" class=\"wp-image-2285\"\/><\/figure>\n\n\n\n<p><\/p>\n\n\n\n<p><\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><strong>\u4e94\u3001\u89c2\u70b9\u56de\u987e<\/strong><\/h2>\n\n\n\n<p>\u6211\u4eec\u5229\u7528\u5b9a\u5236\u7684\u4ea4\u6613\u73af\u5883\uff0c\u5b8c\u6210\u4e86\u4e3a\u80a1\u7968\u4ea4\u6613\u8bbe\u7f6e\u548c\u8bad\u7ec3\u5f3a\u5316\u5b66\u4e60\u4ee3\u7406\u7684\u590d\u6742\u8fc7\u7a0b\u3002\u6211\u4eec\u9996\u5148\u8bbe\u8ba1\u4e86\u4e00\u4e2a\u5168\u9762\u7684\u73af\u5883\uff0c\u4ee5\u6355\u6349\u80a1\u7968\u4ea4\u6613\u7684\u7ec6\u5fae\u5dee\u522b\uff0c\u5305\u62ec\u4ea4\u6613\u6210\u672c\u3001\u72b6\u6001\u89c2\u5bdf\u548c\u5956\u52b1\u8ba1\u7b97\u3002\u6709\u4e86\u8fd9\u4e2a\u73af\u5883\uff0c\u6211\u4eec\u8bad\u7ec3\u4e86\u5404\u79cd\u5f3a\u5316\u5b66\u4e60\u4ee3\u7406&#8211;PPO\u3001A2C\u3001DDPG\u3001SAC \u548c TD3&#8211;\u6bcf\u4e2a\u4ee3\u7406\u90fd\u4e3a\u4ea4\u6613\u7b56\u7565\u8d21\u732e\u4e86\u81ea\u5df1\u7684\u72ec\u7279\u4f18\u52bf\u3002\u6211\u4eec\u8fd8\u5b9e\u65bd\u4e86\u4e00\u4e2a\u96c6\u5408\u4ee3\u7406\uff0c\u5c06\u6240\u6709\u5355\u4e2a\u6a21\u578b\u7684\u9884\u6d4b\u7ed3\u5408\u5728\u4e00\u8d77\uff0c\u65e8\u5728\u6700\u5927\u9650\u5ea6\u5730\u63d0\u9ad8\u6027\u80fd\u548c\u7a33\u5065\u6027\u3002<\/p>\n\n\n\n<p>\u6211\u4eec\u7684\u63a2\u7d22\u5c55\u793a\u4e86\u5982\u4f55\u5c06\u8fd9\u4e9b\u5148\u8fdb\u7b97\u6cd5\u5e94\u7528\u4e8e\u73b0\u5b9e\u4e16\u754c\u7684\u4ea4\u6613\u573a\u666f\uff0c\u7a81\u51fa\u4e86\u5b83\u4eec\u6839\u636e\u5e02\u573a\u6570\u636e\u8fdb\u884c\u8c03\u6574\u5e76\u505a\u51fa\u660e\u667a\u51b3\u7b56\u7684\u6f5c\u529b\u3002\u4ece\u8fd9\u4e00\u5b9e\u8df5\u4e2d\u83b7\u5f97\u7684\u542f\u793a\u4e0d\u4ec5\u5c55\u793a\u4e86\u5f3a\u5316\u5b66\u4e60\u5728\u91d1\u878d\u9886\u57df\u7684\u5a01\u529b\uff0c\u8fd8\u5f3a\u8c03\u4e86\u4e25\u683c\u8bc4\u4f30\u548c\u53ef\u89c6\u5316\u5728\u8bc4\u4f30\u4ee3\u7406\u6027\u80fd\u65b9\u9762\u7684\u91cd\u8981\u6027\u3002\u901a\u8fc7\u4e0d\u65ad\u5b8c\u5584\u6211\u4eec\u7684\u6a21\u578b\u5e76\u5206\u6790\u5176\u7ed3\u679c\uff0c\u6211\u4eec\u53ef\u4ee5\u52aa\u529b\u5236\u5b9a\u66f4\u6709\u6548\u7684\u4ea4\u6613\u7b56\u7565\uff0c\u5e76\u52a0\u6df1\u5bf9\u5e02\u573a\u52a8\u6001\u7684\u7406\u89e3\u3002<\/p>\n\n\n\n<p>\u6df1\u5ea6\u5f3a\u5316\u5b66\u4e60\u5728\u81ea\u52a8\u80a1\u7968\u4ea4\u6613\u7cfb\u7edf\u4e2d\u7684\u5e94\u7528\u53ef\u4ee5\u901a\u8fc7\u5b66\u4e60\u548c\u4f18\u5316\u4ea4\u6613\u7b56\u7565\u6765\u63d0\u9ad8\u4ea4\u6613\u6536\u76ca\u3002\u901a\u8fc7\u7ed3\u5408\u6280\u672f\u6307\u6807\u548cDRL\u7b97\u6cd5\uff0c\u53ef\u4ee5\u5f00\u53d1\u51fa\u80fd\u591f\u9002\u5e94\u5e02\u573a\u53d8\u5316\u5e76\u505a\u51fa\u660e\u667a\u4ea4\u6613\u51b3\u7b56\u7684\u667a\u80fd\u4ea4\u6613\u7cfb\u7edf\u3002\u6b64\u5916\uff0c\u901a\u8fc7\u4f7f\u7528\u96c6\u5408\u65b9\u6cd5\u548c\u4e25\u683c\u7684\u8bc4\u4f30\u6807\u51c6\uff08\u5982\u590f\u666e\u6bd4\u7387\uff09\uff0c\u53ef\u4ee5\u8fdb\u4e00\u6b65\u63d0\u9ad8\u4ea4\u6613\u7b56\u7565\u7684\u9c81\u68d2\u6027\u548c\u6027\u80fd\u3002\u7f51\u9875\u5f3a\u8c03\u4e86\u5728\u5b9e\u9645\u4ea4\u6613\u4e2d\u5bf9\u6a21\u578b\u6027\u80fd\u8fdb\u884c\u8be6\u7ec6\u5206\u6790\u548c\u53ef\u89c6\u5316\u7684\u91cd\u8981\u6027\uff0c\u4ee5\u53ca\u6301\u7eed\u4f18\u5316\u6a21\u578b\u4ee5\u9002\u5e94\u5e02\u573a\u52a8\u6001\u7684\u5fc5\u8981\u6027\u3002<\/p>\n\n\n\n<p><strong>\u611f\u8c22\u60a8\u9605\u8bfb\u5230\u6700\u540e\u3002\u5982\u679c\u5bf9\u6587\u4e2d\u7684\u5185\u5bb9\u6709\u4efb\u4f55\u7591\u95ee\uff0c\u8bf7\u7ed9\u6211\u7559\u8a00\uff0c\u5fc5\u590d\u3002<\/strong><\/p>\n\n\n\n<p><\/p>\n\n\n\n<p><\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<p><\/p>\n\n\n\n<p><\/p>\n\n\n\n<p class=\"has-text-align-center\"><strong><mark style=\"background-color:#ffffff\" class=\"has-inline-color has-cyan-bluish-gray-color\">\u672c<\/mark><\/strong><strong><mark style=\"background-color:#ffffff\" class=\"has-inline-color has-cyan-bluish-gray-color\">\u6587\u5185\u5bb9\u4ec5\u4ec5\u662f\u6280\u672f\u63a2\u8ba8\u548c\u5b66\u4e60\uff0c\u5e76\u4e0d\u6784\u6210\u4efb\u4f55\u6295\u8d44\u5efa\u8bae\u3002<\/mark><\/strong><\/p>\n\n\n\n<p class=\"has-text-align-center\"><strong><mark style=\"background-color:#ffffff\" class=\"has-inline-color has-cyan-bluish-gray-color\">\u8f6c\u53d1\u8bf7\u6ce8\u660e\u539f\u4f5c\u8005\u548c\u51fa\u5904\u3002<\/mark><\/strong><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u4f5c\u8005\uff1a\u8001\u4f59\u635e\u9c7c \u539f\u521b\u4e0d\u6613\uff0c\u8f6c\u8f7d\u8bf7\u6807\u660e\u51fa\u5904\u53ca\u539f\u4f5c\u8005\u3002&#8230;<\/p>\n<div class=\"more-link-wrapper\"><a class=\"more-link\" href=\"https:\/\/laoyulaoyu.com\/index.php\/2024\/10\/24\/%e8%a7%a3%e5%af%86%ef%bc%81%e6%b7%b1%e5%ba%a6%e5%bc%ba%e5%8c%96%e5%ad%a6%e4%b9%a0%e5%a6%82%e4%bd%95%e5%ae%9e%e7%8e%b0%e8%87%aa%e5%8a%a8%e8%82%a1%e7%a5%a8%e4%ba%a4%e6%98%93\/\">Continue reading<span class=\"screen-reader-text\">\u89e3\u5bc6\uff01\u6df1\u5ea6\u5f3a\u5316\u5b66\u4e60\u5982\u4f55\u5b9e\u73b0\u81ea\u52a8\u80a1\u7968\u4ea4\u6613<\/span><\/a><\/div>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[2],"tags":[4,5],"class_list":{"0":"post-1540","1":"post","2":"type-post","3":"status-publish","4":"format-standard","5":"hentry","6":"category-aiinvest","7":"tag-ai","9":"entry"},"_links":{"self":[{"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/posts\/1540","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/comments?post=1540"}],"version-history":[{"count":1,"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/posts\/1540\/revisions"}],"predecessor-version":[{"id":1541,"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/posts\/1540\/revisions\/1541"}],"wp:attachment":[{"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/media?parent=1540"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/categories?post=1540"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/laoyulaoyu.com\/index.php\/wp-json\/wp\/v2\/tags?post=1540"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}