diff --git a/pom.xml b/pom.xml index 6ce314d4..9fe28ac2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,14 +4,14 @@ io.github.kingschan1204 istock jar - 1.6.3 + 1.6.7 pritice Maven Webapp http://maven.apache.org org.springframework.boot spring-boot-starter-parent - 1.5.6.RELEASE + 1.5.10.RELEASE diff --git a/readme.md b/readme.md index 19cbcb01..9572949a 100644 --- a/readme.md +++ b/readme.md @@ -16,7 +16,7 @@ ## Demo 效果 -:link: [点我查看效果](http://stock.51so.info/) (demo版) :link: +:link: [点我查看效果](http://211.159.182.106/) (demo版) :link: ## :boom: 效果图 :boom: diff --git a/src/main/java/io.github.kingschan1204.istock/Application.java b/src/main/java/io.github.kingschan1204.istock/Application.java index cdb5e6f8..4f57b87e 100644 --- a/src/main/java/io.github.kingschan1204.istock/Application.java +++ b/src/main/java/io.github.kingschan1204.istock/Application.java @@ -1,6 +1,7 @@ package io.github.kingschan1204.istock; import io.github.kingschan1204.istock.module.startup.InitQuartzTaskRunner; +import org.springframework.boot.Banner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cache.annotation.EnableCaching; @@ -8,7 +9,10 @@ import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; - +/** + * spring boot 启动类 + * @author kings.chan + */ @Controller @EnableCaching @SpringBootApplication @@ -26,6 +30,8 @@ public InitQuartzTaskRunner startupRunner() { } public static void main(String[] args) { - SpringApplication.run(Application.class, args); + SpringApplication sa = new SpringApplication(Application.class); + sa.setBannerMode(Banner.Mode.OFF); + sa.run( args); } } \ No newline at end of file diff --git a/src/main/java/io.github.kingschan1204.istock/common/conf/RestClientConfig.java b/src/main/java/io.github.kingschan1204.istock/common/conf/RestClientConfig.java new file mode 100644 index 00000000..dc6d1bea --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/common/conf/RestClientConfig.java @@ -0,0 +1,78 @@ +package io.github.kingschan1204.istock.common.conf; + +import org.apache.http.client.HttpClient; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.TrustStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.DefaultResponseErrorHandler; +import org.springframework.web.client.RestTemplate; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +/** + * restTemplate 配置 + * @author chenguoxiang + * @create 2018-10-30 15:56 + **/ +@Configuration +public class RestClientConfig { + private static Logger log = LoggerFactory.getLogger(RestClientConfig.class); + + @Bean + public RestTemplate restTemplate() { + RestTemplate restTemplate = new RestTemplate(); + restTemplate.setRequestFactory(clientHttpRequestFactory()); + restTemplate.setErrorHandler(new DefaultResponseErrorHandler()); + return restTemplate; + } + @Bean + public HttpComponentsClientHttpRequestFactory clientHttpRequestFactory() { + try { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { + public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException { + return true; + } + }).build(); + httpClientBuilder.setSSLContext(sslContext); + HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE; + SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier); + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.getSocketFactory()) + .register("https", sslConnectionSocketFactory).build();// 注册http和https请求 + // 开始设置连接池 + PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + poolingHttpClientConnectionManager.setMaxTotal(500); // 最大连接数500 + poolingHttpClientConnectionManager.setDefaultMaxPerRoute(100); // 同路由并发数100 + httpClientBuilder.setConnectionManager(poolingHttpClientConnectionManager); + httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(3, true)); // 重试次数 + HttpClient httpClient = httpClientBuilder.build(); + HttpComponentsClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory(httpClient); // httpClient连接配置 + clientHttpRequestFactory.setConnectTimeout(20000); // 连接超时 + clientHttpRequestFactory.setReadTimeout(30000); // 数据读取超时时间 + clientHttpRequestFactory.setConnectionRequestTimeout(20000); // 连接不够用的等待时间 + return clientHttpRequestFactory; + } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { + log.error("初始化HTTP连接池出错{}", e); + } + return null; + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/quartz/QuartzManager.java b/src/main/java/io.github.kingschan1204.istock/common/util/quartz/QuartzManager.java index 967b6588..cfd8d0c5 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/quartz/QuartzManager.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/quartz/QuartzManager.java @@ -31,8 +31,7 @@ public class QuartzManager { public void addJob(String jobName, String jobGroupName, String triggerName, String triggerGroupName, Class jobClass, String cron) { try { - Scheduler sched = schedulerFactoryBean.getScheduler();//schedulerFactory.getScheduler(); - + Scheduler sched = schedulerFactoryBean.getScheduler(); // 任务名，任务组，任务执行类 JobDetail jobDetail = JobBuilder.newJob(jobClass).withIdentity(jobName, jobGroupName).build(); // 触发器 diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockDateUtil.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockDateUtil.java index 54ea43c8..55019bf7 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockDateUtil.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockDateUtil.java @@ -10,7 +10,9 @@ import java.util.Date; /** - * Created by kingschan on 2017/6/29. + * 代码定时更新任务 + * @author kings.chan + * @date 2018-6-29 */ public class StockDateUtil { @@ -111,8 +113,9 @@ public static long getCurrentTimestamp() { */ public static long dateToUnixTime(String dateString) throws ParseException { Date date1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") - .parse(dateString);// HH:mm:ss - long temp = date1.getTime();// JAVA的时间戳长度是13位 + .parse(dateString); + // JAVA的时间戳长度是13位 + long temp = date1.getTime(); return temp / 1000; } diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockSpider.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockSpider.java index 85e6067e..8e3aadf2 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockSpider.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockSpider.java @@ -26,7 +26,7 @@ **/ public interface StockSpider { - static final String REGEX_NUMBER = "^[-+]?([0]{1}(\\.[0-9]+)?|[1-9]{1}\\d*(\\.[0-9]+)?)";//"^[-+]?[0-9]+(\\.[0-9]+)?$"; + static final String REGEX_NUMBER = "^[-+]?([0]{1}(\\.[0-9]+)?|[1-9]{1}\\d*(\\.[0-9]+)?)"; /** * 将股票代码转换成新浪接口的格式http://hq.sinajs.cn/list= @@ -80,9 +80,10 @@ static String findStrByRegx(String text, String regx) { /** * 是否工作日 * @param date - * @return + * @return true 工作日 false 非工作日 + * @throws IOException */ - static boolean isWorkDay(String date) throws IOException { + static boolean isWorkDay(Integer date) throws IOException { String api =String.format("http://api.goseek.cn/Tools/holiday?date=%s",date); String result = Jsoup.connect(api).timeout(3000).ignoreContentType(true).get().text(); //{"code":10001,"data":2} 工作日对应结果为 0, 休息日对应结果为 1, 节假日对应的结果为 2 @@ -156,9 +157,9 @@ public void checkServerTrusted(X509Certificate[] certs, String authType) { /** * 得到指定代码的价格 - * * @param stockCode * @return + * @throws Exception */ JSONArray getStockPrice(String[] stockCode) throws Exception; diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/DefaultSpiderImpl.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/DefaultSpiderImpl.java index 7bfe3485..6bd057e2 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/DefaultSpiderImpl.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/DefaultSpiderImpl.java @@ -17,12 +17,9 @@ import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; - import java.io.File; import java.math.RoundingMode; import java.net.SocketTimeoutException; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; import java.text.NumberFormat; import java.util.*; @@ -38,7 +35,10 @@ public class DefaultSpiderImpl implements StockSpider { private static Logger log = LoggerFactory.getLogger(DefaultSpiderImpl.class); @Value("${spider.timeout}") - protected int timeout;//8s超时 + /** + * 8s超时 + */ + protected int timeout; @Value("${spider.useagent}") protected String useAgent; @Value("${xueqiu.token}") @@ -79,24 +79,33 @@ public JSONArray getStockPrice(String[] stockCode) throws Exception { double todayMax = StockSpider.mathFormat(data[5]); double todayMin = StockSpider.mathFormat(data[6]); json = new JSONObject(); - if (xj == 0) { //一般这种是停牌的 - json.put("fluctuate", 0);//波动 + //一般这种是停牌的 + if (xj == 0) { + //波动 + json.put("fluctuate", 0); } else { NumberFormat nf = NumberFormat.getNumberInstance(); // 保留两位小数 nf.setMaximumFractionDigits(2); // 如果不需要四舍五入，可以使用RoundingMode.DOWN nf.setRoundingMode(RoundingMode.UP); - json.put("fluctuate", StockSpider.mathFormat(nf.format(zf)));//波动 + //波动 + json.put("fluctuate", StockSpider.mathFormat(nf.format(zf))); } TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai")); - json.put("code", data[0]);//代码 + //代码 + json.put("code", data[0]); json.put("type", StockSpider.formatStockCode(data[0]).replaceAll("\\d", "")); - json.put("name", data[1].replaceAll("\\s", ""));//名称 - json.put("price", xj);//现价 - json.put("todayMax", todayMax);//今日最高价 - json.put("todayMin", todayMin);//今日最低价 - json.put("yesterdayPrice", zs);//昨收 + //名称 + json.put("name", data[1].replaceAll("\\s", "")); + //现价 + json.put("price", xj); + //今日最高价 + json.put("todayMax", todayMax); + //今日最低价 + json.put("todayMin", todayMin); + //昨收 + json.put("yesterdayPrice", zs); json.put("priceDate", StockDateUtil.getCurrentDateTimeNumber()); rows.add(json); } @@ -112,27 +121,43 @@ public JSONObject getStockInfo(String code) throws Exception { Elements table = doc.getElementsByTag("table"); //第一个表格的第一行 Elements tds = table.get(0).select("tr").get(0).select("td"); - String zyyw = tds.get(0).text().replaceAll(regex, "");//主营业务 - String sshy = tds.get(1).text().replaceAll(regex, "");//所属行业 + //主营业务 + String zyyw = tds.get(0).text().replaceAll(regex, ""); + //所属行业 + String sshy = tds.get(1).text().replaceAll(regex, ""); Elements tds1 = table.get(1).select("td"); - String dtsyl = tds1.get(0).text().replaceAll(regex, "");//市盈率(动态) - String sjljt = tds1.get(4).text().replaceAll(regex, "");//市盈率(静态) - String sjl = tds1.get(8).text().replaceAll(regex, "");//市净率 - String zsz = tds1.get(11).text().replaceAll("\\D+", "");//总市值 - double mgjzc = StockSpider.mathFormat(tds1.get(12).text().replaceAll("\\[.*|", ""));//每股净资产 + //市盈率(动态) + String dtsyl = tds1.get(0).text().replaceAll(regex, ""); + //市盈率(静态) + String sjljt = tds1.get(4).text().replaceAll(regex, ""); + //市净率 + String sjl = tds1.get(8).text().replaceAll(regex, ""); + //总市值 + String zsz = tds1.get(11).text().replaceAll("\\D+", ""); + //每股净资产 + double mgjzc = StockSpider.mathFormat(tds1.get(12).text().replaceAll("\\[.*|", "")); String jzcsyl = "-1"; if (tds1.size() > 14) { - jzcsyl = tds1.get(14).select("span").get(1).text();//净资产收益率 + //净资产收益率 + jzcsyl = tds1.get(14).select("span").get(1).text(); } JSONObject json = new JSONObject(); - json.put("mainBusiness", zyyw);//主营业务 - json.put("industry", sshy);//所属行业 - json.put("ped", StockSpider.mathFormat(dtsyl));//市盈率(动态) - json.put("pes", StockSpider.mathFormat(sjljt));//市盈率(静态) - json.put("pb", StockSpider.mathFormat(sjl));//市净率 - json.put("totalValue", StockSpider.mathFormat(zsz));//总市值 - json.put("roe", StockSpider.mathFormat(jzcsyl));//净资产收益率 - json.put("bvps", mgjzc);//每股净资产 + //主营业务 + json.put("mainBusiness", zyyw); + //所属行业 + json.put("industry", sshy); + //市盈率(动态) + json.put("ped", StockSpider.mathFormat(dtsyl)); + //市盈率(静态) + json.put("pes", StockSpider.mathFormat(sjljt)); + //市净率 + json.put("pb", StockSpider.mathFormat(sjl)); + //总市值 + json.put("totalValue", StockSpider.mathFormat(zsz)); + //净资产收益率 + json.put("roe", StockSpider.mathFormat(jzcsyl)); + //每股净资产 + json.put("bvps", mgjzc); json.put("infoDate", StockDateUtil.getCurrentDateNumber()); json.put("code", stockCode); json.put("type", StockSpider.formatStockCode(stockCode).replaceAll("\\d", "")); @@ -160,16 +185,26 @@ public JSONArray getHistoryDividendRate(String code) throws Exception { log.debug("报告期:{},A股除权除息日:{},实施日期:{},分红方案说明:{},分红率:{}", data[0], data[6], data[3], data[4], data[9]); json = new JSONObject(); json.put("code", stockCode); - json.put("title", data[0]);//报告期 - json.put("releaseDate", data[1]);//披露时间 董事会日期 - json.put("plan", data[4]);//分红方案 - json.put("sgbl", 0);//送股比例 - json.put("zgbl", 0);//转股比例 - json.put("percent", StockSpider.mathFormat(data[9]));//分红率 - json.put("gqdjr", data[5]);//股权登记日 - json.put("cxcqr", data[6]);//除息除权日 - json.put("progress", data[7]);//方案进度 - json.put("from", "ths");//来源 + //报告期 + json.put("title", data[0]); + //披露时间 董事会日期 + json.put("releaseDate", data[1]); + //分红方案 + json.put("plan", data[4]); + //送股比例 + json.put("sgbl", 0); + //转股比例 + json.put("zgbl", 0); + //分红率 + json.put("percent", StockSpider.mathFormat(data[9])); + //股权登记日 + json.put("gqdjr", data[5]); + //除息除权日 + json.put("cxcqr", data[6]); + //方案进度 + json.put("progress", data[7]); + //来源 + json.put("from", "ths"); jsons.add(json); } return jsons; @@ -180,7 +215,7 @@ public JSONArray getHistoryDividendRate(String code) throws Exception { @Override public JSONArray getHistoryROE(String code) throws Exception { String url = String.format("http://basic.10jqka.com.cn/api/stock/export.php?export=main&type=year&code=%s", code); - String path = String.format("./data/%s_main_year.xls", code); + String path = String.format("./data/%s.xls", code); String referrer=String.format("http://basic.10jqka.com.cn/%s/finance.html",code); if (!new File(path).exists()) { //下载 @@ -190,8 +225,11 @@ public JSONArray getHistoryROE(String code) throws Exception { } //读取excel数据 List list = ExcelOperactionTool.readExcelData(path); + //报告期 年 Object[] year = list.get(1); + //净资产收益率 Object[] roe = list.get(10); + //净资产收益率-摊薄 Object[] roeTb = list.get(11); JSONArray jsons = new JSONArray(); JSONObject json; diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/EastmoneySpider.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/EastmoneySpider.java index 2f95d160..9e7acadb 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/EastmoneySpider.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/EastmoneySpider.java @@ -73,16 +73,26 @@ public JSONArray getHistoryDividendRate(String code) throws Exception { if (title.matches("^\\d{4}\\-06-30$")) { title = title.replaceAll("\\-.*", "") + "中报"; } - temp.put("title", title);//报告期 - temp.put("releaseDate", item.getString("ResultsbyDate").replaceAll(regex, ""));//披露时间 - temp.put("plan", item.getString("AllocationPlan"));//分配预案 - temp.put("sgbl", intFormart(item.getString("SGBL")));//送股比例 - temp.put("zgbl", intFormart(item.getString("ZGBL")));//转股比例 - temp.put("percent", doubleFormat(item.getString("GXL"), true));//股息率 - temp.put("gqdjr", item.getString("GQDJR").replaceAll(regex, ""));//股权登记日 - temp.put("cxcqr", item.getString("CQCXR").replaceAll(regex, ""));//除息除权日 - temp.put("progress", item.getString("ProjectProgress"));//进度 - temp.put("from", "east");//来源 + //报告期 + temp.put("title", title); + //披露时间 + temp.put("releaseDate", item.getString("ResultsbyDate").replaceAll(regex, "")); + //分配预案 + temp.put("plan", item.getString("AllocationPlan")); + //送股比例 + temp.put("sgbl", intFormart(item.getString("SGBL"))); + //转股比例 + temp.put("zgbl", intFormart(item.getString("ZGBL"))); + //股息率 + temp.put("percent", doubleFormat(item.getString("GXL"), true)); + //股权登记日 + temp.put("gqdjr", item.getString("GQDJR").replaceAll(regex, "")); + //除息除权日 + temp.put("cxcqr", item.getString("CQCXR").replaceAll(regex, "")); + //进度 + temp.put("progress", item.getString("ProjectProgress")); + //来源 + temp.put("from", "east"); jsons.add(temp); } diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/JisiluSpilder.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/JisiluSpilder.java index 72c2fb7f..8bc6bb16 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/JisiluSpilder.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/JisiluSpilder.java @@ -31,9 +31,7 @@ public class JisiluSpilder extends DefaultSpiderImpl { * @return */ public JSONObject crawHisPbPePriceAndReports(String code) throws Exception { -// String useAgent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3346.9 Safari/537.36"; String referrer = "https://www.jisilu.cn/data/stock/dividend_rate/#cn"; -// int timeout=8000; JSONObject result = new JSONObject(); String url = String.format("https://www.jisilu.cn/data/stock/%s", code); log.info("craw jisilu page :{}", url); @@ -71,15 +69,30 @@ public JSONObject crawHisPbPePriceAndReports(String code) throws Exception { List list = StockSpider.findStringByRegx(js.html(), "\\[.*\\]"); //依次顺序 0:日期 1:PRICE 2:PB 3:PE String replaceRegex = "\'|\\[|\\]"; - String dates[] = list.get(0).replaceAll(replaceRegex, "").split(",");//日期 - String prices[] = list.get(1).replaceAll(replaceRegex, "").split(",");//价格 - String pbs[] = list.get(2).replaceAll(replaceRegex, "").split(",");//市净率 - String pes[] = list.get(3).replaceAll(replaceRegex, "").split(",");//市盈率 + //日期 + String dates[] = list.get(0).replaceAll(replaceRegex, "").split(","); + //价格 + String prices[] = list.get(1).replaceAll(replaceRegex, "").split(","); + //市净率 + String pbs[] = list.get(2).replaceAll(replaceRegex, "").split(","); + //市盈率 + String pes[] = list.get(3).replaceAll(replaceRegex, "").split(","); JSONArray hisJson = new JSONArray(); for (int i = 0; i < dates.length; i++) { + double price=Double.parseDouble(prices[i]); + if(i!=0){ + double lastPrice=Double.parseDouble(prices[i-1]); + //如果价格 昨天和今天大于，小于 10%的幅度则为脏数据 + double maxPrice=lastPrice+(lastPrice*0.1); + double minPrice=lastPrice-(lastPrice*0.1); + if(price>maxPrice||price rows =Arrays.asList(content.split(";")); + JSONArray jsonArray = new JSONArray(); + JSONObject json; + for (int i = 0; i < rows.size(); i++) { + String [] item =rows.get(i).replaceAll("v_.*=|\"","").split("~"); + json= new JSONObject(); + //代码 + json.put("code", item[2]); + //波动 + json.put("fluctuate", item[32]); + json.put("type", StockSpider.formatStockCode(item[2]).replaceAll("\\d", "")); + //名称 + json.put("name", item[1].replaceAll("\\s", "")); + //现价 + json.put("price", item[3]); + //今日最高价 + json.put("todayMax", item[41]); + //今日最低价 + json.put("todayMin", item[42]); + //昨收 + json.put("yesterdayPrice", item[4]); + json.put("priceDate", StockDateUtil.getCurrentDateTimeNumber()); + jsonArray.add(json); + } + return jsonArray; + } catch (SocketTimeoutException e) { + e.printStackTrace(); + return null; + } + } + + +} diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/TushareSpider.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/TushareSpider.java new file mode 100644 index 00000000..5bc8e0eb --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/TushareSpider.java @@ -0,0 +1,154 @@ +package io.github.kingschan1204.istock.common.util.stock.impl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import io.github.kingschan1204.istock.common.util.stock.StockSpider; +import org.jsoup.Jsoup; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; + +/** + * + * 用挖地免的接口 + * @author chenguoxiang + * @create 2018-10-30 9:31 + **/ +@RefreshScope +@Component("TushareSpider") +public class TushareSpider { + + @Value("${tushare.token}") + private String tuToken; + @Autowired + private RestTemplate restTemplate; + final String api="http://api.tushare.pro"; + + /** + * 格式化成tushare API所需格式 + * @param code + * @return + */ + public static String formatCode(String code) { + //5开头，沪市基金或权证 60开头上证 + if (code.matches("^60.*|^5.*")) { + return String.format("%s.SH", code); + } + //1开头的，是深市基金 00开头是深圳 + else if (code.matches("^1.*|^00.*|^300...")) { + return String.format("%s.SZ", code); + } + return null; + } + + /** + * post 方式提交 + * @param params + * @return + */ + String post(JSONObject params){ + HttpHeaders headers = new HttpHeaders(); + //定义请求参数类型，这里用json所以是MediaType.APPLICATION_JSON + headers.setContentType(MediaType.APPLICATION_JSON); + params.put("token", tuToken); + HttpEntity formEntity = new HttpEntity(params.toString(), headers); + String result = restTemplate.postForObject(api, formEntity, String.class); + return result; + } + + + /** + * 返回已上市的A股代码 + * @return TS代码,股票代码,股票名称,所在地域,所属行业,股票全称,市场类型 （主板/中小板/创业板）,上市状态： L上市 D退市 P暂停上市,上市日期 + */ + public JSONArray getStockCodeList(){ + JSONObject json = new JSONObject(); + //接口名称 + json.put("api_name","stock_basic"); + //只取上市的 + json.put("params",JSON.parse("{'list_status':'L'}")); + json.put("fields","ts_code,symbol,name,area,industry,fullname,market,list_status,list_date"); + String result = post(json); + JSONObject datas= JSON.parseObject(result); + JSONArray items =datas.getJSONObject("data").getJSONArray("items"); + return items; + } + + /** + *获取上海公司基础信息 + * @return + */ + public JSONArray getStockShCompany(){ + JSONObject json = new JSONObject(); + //接口名称 + json.put("api_name","stock_company"); + json.put("params",JSON.parse("{'exchange':'SSE'}")); + json.put("fields","ts_code,chairman,manager,secretary,reg_capital,setup_date,province,city,introduction,website,email,office,employees,main_business,business_scope"); + String result = post(json); + JSONObject data= JSON.parseObject(result); + JSONArray items =data.getJSONObject("data").getJSONArray("items"); + return items; + } + + /** + *获取深圳公司基础信息 + * @return + */ + public JSONArray getStockSZCompany(){ + JSONObject json = new JSONObject(); + //接口名称 + json.put("api_name","stock_company"); + json.put("params",JSON.parse("{'exchange':'SZSE'}")); + json.put("fields","ts_code,chairman,manager,secretary,reg_capital,setup_date,province,city,introduction,website,email,office,employees,main_business,business_scope"); + String result = post(json); + JSONObject data= JSON.parseObject(result); + JSONArray items =data.getJSONObject("data").getJSONArray("items"); + return items; + } + + /** + * 得到前10大持有人 + * @param code + * @return + */ + public JSONArray getStockTopHolders(String code){ + JSONObject json = new JSONObject(); + //接口名称 + json.put("api_name","top10_holders"); + json.put("params",JSON.parse(String.format("{'ts_code':'%s'}",code))); + json.put("fields","ts_code,ann_date,end_date,holder_name,hold_amount,hold_ratio"); + String result = post(json); + JSONObject datas= JSON.parseObject(result); + JSONArray items =datas.getJSONObject("data").getJSONArray("items"); + return items; + } + public static void main(String[] args) { + try { + //{"date":"2018-11-01 18:06:25","code":200,"address":"湖南省长沙市 电信","ip":"113.246.64.67"} + System.setProperty("https.maxRedirects", "50"); + System.getProperties().setProperty("https.proxySet", "true"); + System.getProperties().setProperty("https.proxyHost", "203.86.26.9"); + System.getProperties().setProperty("https.proxyPort", "3128"); + StockSpider.enableSSLSocket(); + String url ="https://api.ttt.sh/ip/qqwry/"; + String json =Jsoup.connect(url).get().text(); + JSONObject j = JSON.parseObject(json); + System.out.println(j); + } catch (IOException e) { + e.printStackTrace(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } catch (KeyManagementException e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockCtrl.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockCtrl.java index 8b1a0760..fcde4a43 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockCtrl.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockCtrl.java @@ -1,25 +1,16 @@ package io.github.kingschan1204.istock.module.maindata.ctrl; -import com.mongodb.BasicDBObject; -import com.mongodb.WriteResult; import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; import io.github.kingschan1204.istock.common.util.stock.StockSpider; -import io.github.kingschan1204.istock.module.maindata.services.StockCodeService; +import io.github.kingschan1204.istock.module.maindata.services.StockCodeInfoService; +import io.github.kingschan1204.istock.module.maindata.services.StockCompanyService; import io.github.kingschan1204.istock.module.maindata.services.StockService; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.mapreduce.MapReduceOptions; -import org.springframework.data.mongodb.core.mapreduce.MapReduceResults; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.mongodb.core.query.Update; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; -import java.util.Iterator; - /** * @author chenguoxiang * @create 2018-03-27 11:14 @@ -30,7 +21,9 @@ public class StockCtrl { @Autowired private StockService service; @Autowired - private StockCodeService stockCodeService; + private StockCodeInfoService stockCodeInfoService; + @Autowired + private StockCompanyService stockCompanyService; @Autowired private StockSpider spider; @@ -43,10 +36,22 @@ public String queryStock(Integer page, Integer rows, String code, String type, S @ResponseBody - @RequestMapping(value = "/stock/init_code", method = RequestMethod.POST) + @RequestMapping(value = "/stock/refresh_code", method = RequestMethod.POST) public String initCode() { try { - stockCodeService.saveAllStockCode(); + stockCodeInfoService.refreshCode(); + return "success"; + } catch (Exception e) { + e.printStackTrace(); + return e.getMessage(); + } + } + + @ResponseBody + @RequestMapping(value = "/stock/refresh_company", method = RequestMethod.POST) + public String initCompany() { + try { + stockCompanyService.refreshStockCompany(); return "success"; } catch (Exception e) { e.printStackTrace(); @@ -54,39 +59,12 @@ public String initCode() { } } - @Autowired - private MongoTemplate template; @ResponseBody @RequestMapping(value = "/stock/mapReduce/5years_dy") public String fiveYearsDy() { - int year = StockDateUtil.getCurrentYear(); - int fiveYearAgo = year - 5; - String startDate = StockDateUtil.getCurrYearLastDay(fiveYearAgo).toString(); - String endDate = StockDateUtil.getCurrentDate(); - Query query = new Query(); - query.addCriteria(Criteria.where("releaseDate").gte(startDate).lte(endDate)); - MapReduceResults result = template.mapReduce(query, "stock_dividend", - "classpath:/mapreduce/5years_dy/dy5years_map.js", "classpath:/mapreduce/5years_dy/dy5years_reduce.js", - new MapReduceOptions().outputCollection("stock_dy_statistics"), BasicDBObject.class); - Iterator iter = result.iterator(); - while (iter.hasNext()) { - BasicDBObject item = iter.next(); - String code = item.getString("_id"); - BasicDBObject value = (BasicDBObject) item.get("value"); - if (value.containsKey("size") && value.getInt("size") > 4) { - double percent = Double.parseDouble(value.getString("percent")); - WriteResult wr = template.upsert( - new Query(Criteria.where("_id").is(code)), - new Update() - .set("_id", code) - .set("fiveYearDy", percent), - "stock" - ); - } - - } - return String.format("success:%s - %s", startDate, endDate); + service.calculateFiveYearsDy(); + return "success"; } @@ -95,28 +73,7 @@ public String fiveYearsDy() { public String fiveYearsRoe() { int endYear = StockDateUtil.getCurrentYear(); int startYear = endYear - 5; - Query query = new Query(); - query.addCriteria(Criteria.where("year").gte(startYear).lte(endYear)); - MapReduceResults result = template.mapReduce(query, "stock_his_roe", - "classpath:/mapreduce/5years_roe/map.js", "classpath:/mapreduce/5years_roe/reduce.js", - new MapReduceOptions().outputCollection("stock_hisroe_statistics"), BasicDBObject.class); - Iterator iter = result.iterator(); - while (iter.hasNext()) { - BasicDBObject item = iter.next(); - String code = item.getString("_id"); - BasicDBObject value = (BasicDBObject) item.get("value"); - if (value.containsKey("size") && value.getInt("size") > 4) { - double percent = Double.parseDouble(value.getString("percent")); - WriteResult wr = template.upsert( - new Query(Criteria.where("_id").is(code)), - new Update() - .set("_id", code) - .set("fiveYearRoe", percent), - "stock" - ); - } - - } + service.calculateFiveYearsRoe(startYear,endYear); return String.format("success:%s - %s", startYear, endYear); } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockHisPageCtrl.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockHisPageCtrl.java index 6e4c0872..60a9ebf9 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockHisPageCtrl.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockHisPageCtrl.java @@ -121,7 +121,7 @@ public ModelAndView getStockHisRoe(@PathVariable String code) { @Autowired private MongoTemplate template; - @RequestMapping("/stock/info/{code}") + @RequestMapping("/stock/hisdata/{code}") public ModelAndView getStockInfo(@PathVariable String code) throws Exception { ModelAndView mav = new ModelAndView(template_path + "his_pbpe"); StringBuilder date = new StringBuilder(); diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockInfoCtl.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockInfoCtl.java new file mode 100644 index 00000000..d43ae872 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockInfoCtl.java @@ -0,0 +1,34 @@ +package io.github.kingschan1204.istock.module.maindata.ctrl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import io.github.kingschan1204.istock.module.maindata.services.StockInfoService; +import io.github.kingschan1204.istock.module.maindata.vo.StockVo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; + +/** + * @author chenguoxiang + * @create 2018-11-02 15:49 + **/ +@Controller +public class StockInfoCtl { + private Logger log = LoggerFactory.getLogger(StockInfoCtl.class); + @Autowired + private StockInfoService stockInfoService; + + @RequestMapping("/stock/info/{code}") + public String stockInfo(@PathVariable String code, Model model) { + JSONObject json= stockInfoService.getStockInfo(code); + StockVo stockVo = JSON.toJavaObject(json,StockVo.class); + model.addAttribute("pagetitle",String.format("%s-%s",code,stockVo.getName())); + model.addAttribute("data",json.toJSONString()); + model.addAttribute("stock", JSON.toJSONString(stockVo)); + return "/stock/info/stock_info"; + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/Stock.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/Stock.java index 5c34342a..74a734f0 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/Stock.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/Stock.java @@ -5,7 +5,6 @@ /** * stock maindata - * * @author chenguoxiang * @create 2018-03-27 10:15 **/ @@ -31,13 +30,31 @@ public class Stock { private Double pes; private Double ped; private Integer infoDate; - private String dividendDate;//静态分红日期 + /** + * 静态分红日期 + */ + private String dividendDate; private Double dividend; - private Integer dividendUpdateDay;//静态分红更新时间 - private Double dy;//实时股息率 - private Integer dyDate;//实时股息更新时间 - private Double fiveYearDy;//5年平均股息 - private Double fiveYearRoe;//5年平均Roe + /** + * 静态分红更新时间 + */ + private Integer dividendUpdateDay; + /** + * 实时股息率 + */ + private Double dy; + /** + * 实时股息更新时间 + */ + private Integer dyDate; + /** + * 5年平均股息 + */ + private Double fiveYearDy; + /** + * 5年平均Roe + */ + private Double fiveYearRoe; public String getCode() { diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCode.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCode.java deleted file mode 100644 index 6f35a346..00000000 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCode.java +++ /dev/null @@ -1,50 +0,0 @@ -package io.github.kingschan1204.istock.module.maindata.po; - -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -/** - * stock code - * @author chenguoxiang - * @create 2018-04-08 16:40 - **/ -@Document(collection = "stock_code") -public class StockCode { - - @Id - private String code; - private Integer hrdud;//history report data update date 历史年度报表更新时间 - private Integer xlsError;//下载xls错误 - - public StockCode(){} - public StockCode(String code){ - this.code=code; - this.hrdud=0; - this.xlsError=0; - } - - - public String getCode() { - return code; - } - - public void setCode(String code) { - this.code = code; - } - - public Integer getHrdud() { - return hrdud; - } - - public void setHrdud(Integer hrdud) { - this.hrdud = hrdud; - } - - public Integer getXlsError() { - return xlsError; - } - - public void setXlsError(Integer xlsError) { - this.xlsError = xlsError; - } -} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCodeInfo.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCodeInfo.java new file mode 100644 index 00000000..41f1e447 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCodeInfo.java @@ -0,0 +1,183 @@ +package io.github.kingschan1204.istock.module.maindata.po; + +import com.alibaba.fastjson.JSONArray; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +/** + * 代码信息 + * @author chenguoxiang + * @create 2018-10-30 15:15 + **/ +@Document(collection = "stock_code_info") +public class StockCodeInfo { + + //股票代码 + @Id + private String code; + // 深市 沪市 + private String type; + //股票名称 + private String name; + //所在地域 + private String area; + //所属行业 + private String industry; + //股票全称 + private String fullname; + //市场类型（主板/中小板/创业板） + private String market; + //上市状态L上市 D退市 P暂停上市 + private String list_status; + //上市日期 + private Integer list_date; + + + /** + * info 信息更新日 + */ + private Integer infoDate; + /** + * 持有人更新日 + */ + private Integer holdersDate; + /** + * 年报excel下载时间 + */ + private Integer yearReportDate; + /** + * xls下载错误数 + */ + private Integer xlsError; + + //init method + public StockCodeInfo(){ } + + /** + * data 来源于 tushareSpilder + * @param data + */ + public StockCodeInfo(JSONArray data){ + this.code=data.getString(1); + this.type=data.getString(0).replaceAll("\\d+\\.","").toLowerCase(); + this.name=data.getString(2); + this.area=data.getString(3); + this.industry=data.getString(4); + this.fullname=data.getString(5); + this.market=data.getString(6); + this.list_status=data.getString(7); + this.list_date=data.getInteger(8); + this.infoDate= 0; + this.holdersDate=0; + this.yearReportDate=0; + this.xlsError=0; + + } + + + + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getArea() { + return area; + } + + public void setArea(String area) { + this.area = area; + } + + public String getIndustry() { + return industry; + } + + public void setIndustry(String industry) { + this.industry = industry; + } + + public String getFullname() { + return fullname; + } + + public void setFullname(String fullname) { + this.fullname = fullname; + } + + public String getMarket() { + return market; + } + + public void setMarket(String market) { + this.market = market; + } + + public String getList_status() { + return list_status; + } + + public void setList_status(String list_status) { + this.list_status = list_status; + } + + public Integer getList_date() { + return list_date; + } + + public void setList_date(Integer list_date) { + this.list_date = list_date; + } + + public Integer getInfoDate() { + return infoDate; + } + + public void setInfoDate(Integer infoDate) { + this.infoDate = infoDate; + } + + public Integer getHoldersDate() { + return holdersDate; + } + + public void setHoldersDate(Integer holdersDate) { + this.holdersDate = holdersDate; + } + + public Integer getYearReportDate() { + return yearReportDate; + } + + public void setYearReportDate(Integer yearReportDate) { + this.yearReportDate = yearReportDate; + } + + public Integer getXlsError() { + return xlsError; + } + + public void setXlsError(Integer xlsError) { + this.xlsError = xlsError; + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCompany.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCompany.java new file mode 100644 index 00000000..0ffc5976 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCompany.java @@ -0,0 +1,178 @@ +package io.github.kingschan1204.istock.module.maindata.po; + +import com.alibaba.fastjson.JSONArray; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +/** + * 上市公司信息 + * @author chenguoxiang + * @create 2018-10-31 14:35 + **/ +@Document(collection = "stock_company") +public class StockCompany { + + @Id + private String code; + private String chairman; + private String manager; + private String secretary; + private Double regCapital; + private String setupDate; + private String province; + private String city; + private String introduction; + private String website; + private String email; + private String office; + private Integer employees; + private String mainBusiness; + private String businessScope; + + + public StockCompany(){} + + public StockCompany(JSONArray json){ + this.code=json.getString(0).replaceAll("\\D+",""); + this.chairman=json.getString(1); + this.manager=json.getString(2); + this.secretary=json.getString(3); + this.regCapital=json.getDouble(4); + this.setupDate=json.getString(5); + this.province=json.getString(6); + this.city=json.getString(7); + this.introduction=json.getString(8); + this.website=json.getString(9); + this.email=json.getString(10); + this.office=json.getString(11); + this.mainBusiness=json.getString(12); + this.employees=json.getInteger(13); + if (null!=json.get(14)){ + this.businessScope=json.getString(14).replaceFirst("^主.*\\:",""); + } + + } + + + + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getChairman() { + return chairman; + } + + public void setChairman(String chairman) { + this.chairman = chairman; + } + + public String getManager() { + return manager; + } + + public void setManager(String manager) { + this.manager = manager; + } + + public String getSecretary() { + return secretary; + } + + public void setSecretary(String secretary) { + this.secretary = secretary; + } + + public Double getRegCapital() { + return regCapital; + } + + public void setRegCapital(Double regCapital) { + this.regCapital = regCapital; + } + + public String getSetupDate() { + return setupDate; + } + + public void setSetupDate(String setup_date) { + this.setupDate = setup_date; + } + + public String getProvince() { + return province; + } + + public void setProvince(String province) { + this.province = province; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getIntroduction() { + return introduction; + } + + public void setIntroduction(String introduction) { + this.introduction = introduction; + } + + public String getWebsite() { + return website; + } + + public void setWebsite(String website) { + this.website = website; + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } + + public String getOffice() { + return office; + } + + public void setOffice(String office) { + this.office = office; + } + + public Integer getEmployees() { + return employees; + } + + public void setEmployees(Integer employees) { + this.employees = employees; + } + + public String getMainBusiness() { + return mainBusiness; + } + + public void setMainBusiness(String main_business) { + this.mainBusiness = main_business; + } + + public String getBusinessScope() { + return businessScope; + } + + public void setBusinessScope(String businessScope) { + this.businessScope = businessScope; + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisDividend.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisDividend.java index daa571cd..7355c09a 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisDividend.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisDividend.java @@ -12,7 +12,6 @@ @Document(collection = "stock_his_dividend") public class StockHisDividend { - //{"date":"2017-07-07","code":"600519","year":"2016年报","executeDate":"2017-07-01","remark":"10派67.87元(含税)","percent":1.44} @Id private String id; private String code; diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockReport.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockReport.java index ef4d66b7..eb87988d 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockReport.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockReport.java @@ -4,7 +4,9 @@ import org.springframework.data.mongodb.core.mapping.Document; /** - * Created by kingschan on 2018/7/6. + * 代码定时更新任务 + * @author kings.chan + * @date 2018-7-6 */ @Document(collection = "stock_report") public class StockReport { diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockTopHolders.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockTopHolders.java new file mode 100644 index 00000000..865c9e85 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockTopHolders.java @@ -0,0 +1,91 @@ +package io.github.kingschan1204.istock.module.maindata.po; + +import com.alibaba.fastjson.JSONArray; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +/** + * + * 代码前几名持有人 + * @author chenguoxiang + * @create 2018-11-01 10:47 + **/ +@Document(collection = "stock_top_holders") +public class StockTopHolders { + @Id + private String id; + private String code; + private Integer annDate; + private Integer endDate; + private String holderName; + private Double holdAmount; + private Double holdRatio; + + + public StockTopHolders(){} + public StockTopHolders(JSONArray json){ + this.code=json.getString(0).replaceAll("\\D",""); + this.annDate=json.getInteger(1); + this.endDate=json.getInteger(2); + this.holderName=json.getString(3); + this.holdAmount=json.getDouble(4); + this.holdRatio=json.getDouble(5); + } + + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public Integer getAnnDate() { + return annDate; + } + + public void setAnnDate(Integer annDate) { + this.annDate = annDate; + } + + public Integer getEndDate() { + return endDate; + } + + public void setEndDate(Integer endDate) { + this.endDate = endDate; + } + + public String getHolderName() { + return holderName; + } + + public void setHolderName(String holderName) { + this.holderName = holderName; + } + + public Double getHoldAmount() { + return holdAmount; + } + + public void setHoldAmount(Double holdAmount) { + this.holdAmount = holdAmount; + } + + public Double getHoldRatio() { + return holdRatio; + } + + public void setHoldRatio(Double holdRatio) { + this.holdRatio = holdRatio; + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeInfoService.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeInfoService.java new file mode 100644 index 00000000..395b4fd5 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeInfoService.java @@ -0,0 +1,70 @@ +package io.github.kingschan1204.istock.module.maindata.services; + +import com.alibaba.fastjson.JSONArray; +import io.github.kingschan1204.istock.common.util.stock.impl.TushareSpider; +import io.github.kingschan1204.istock.module.maindata.po.StockCodeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.List; + +/** + * 代码信息服务类 + * @author chenguoxiang + * @create 2018-10-30 15:22 + **/ +@Service +public class StockCodeInfoService { + private Logger log = LoggerFactory.getLogger(StockCodeInfoService.class); + @Autowired + private MongoTemplate mongoTemplate; + @Autowired + private TushareSpider tushareSpider; + + + /** + * 代码列表刷新 + */ + public void refreshCode(){ + List list = new ArrayList(); + JSONArray rows =tushareSpider.getStockCodeList(); + for (int i = 0; i < rows.size(); i++) { + list.add(new StockCodeInfo(rows.getJSONArray(i))); + } + //先删除再新增 + mongoTemplate.dropCollection(StockCodeInfo.class); + mongoTemplate.insertAll(list); + } + + /** + * 返回所有代码 + * + * @return + */ + public List getAllStockCodes() { + return mongoTemplate.find(new Query(), StockCodeInfo.class); + + } + + /** + * 返回沪市代码 + * @return + */ + public List getSHStockCodes() { + return mongoTemplate.find(new Query(Criteria.where("type").is("sh")), StockCodeInfo.class); + + } + + /** + * 返回深市代码 + * @return + */ + public List getSZStockCodes() { + return mongoTemplate.find(new Query(Criteria.where("type").is("sz")), StockCodeInfo.class); + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeService.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeService.java deleted file mode 100644 index 65409e11..00000000 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeService.java +++ /dev/null @@ -1,68 +0,0 @@ -package io.github.kingschan1204.istock.module.maindata.services; - -import io.github.kingschan1204.istock.common.util.stock.StockSpider; -import io.github.kingschan1204.istock.module.maindata.po.StockCode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.mongodb.core.query.Update; -import org.springframework.stereotype.Service; - -import java.util.HashSet; -import java.util.List; - -/** - * ${DESCRIPTION} - * - * @author chenguoxiang - * @create 2018-04-08 17:11 - **/ -@Service -public class StockCodeService { - - private Logger log = LoggerFactory.getLogger(StockCodeService.class); - @Autowired - private StockSpider stockSpider; - @Autowired - private MongoTemplate mongoTemplate; - - /** - * 保存所有代码 - * - * @throws Exception - */ - public void saveAllStockCode() throws Exception { - List szCodes = stockSpider.getStockCodeBySZ(); - List shCodes = stockSpider.getStockCodeBySH(); - List allcodes = stockSpider.getAllStockCode(); - HashSet codes = new HashSet(); - codes.addAll(szCodes); - codes.addAll(shCodes); - codes.addAll(allcodes); - codes.stream().forEach(code -> { - mongoTemplate.upsert( - new Query(Criteria.where("_id").is(code)), - new Update().set("_id", code).set("hrdud",0).set("xlsError",0), - "stock_code" - ); - } - ); - - - } - - - /** - * 返回所有代码 - * - * @return - */ - public List getAllStockCodes() { - return mongoTemplate.find(new Query(), StockCode.class); - - } - -} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCompanyService.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCompanyService.java new file mode 100644 index 00000000..8567362e --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCompanyService.java @@ -0,0 +1,45 @@ +package io.github.kingschan1204.istock.module.maindata.services; + +import com.alibaba.fastjson.JSONArray; +import io.github.kingschan1204.istock.common.util.stock.impl.TushareSpider; +import io.github.kingschan1204.istock.module.maindata.po.StockCompany; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.List; + +/** + * + * @author chenguoxiang + * @create 2018-10-31 16:42 + **/ +@Service +public class StockCompanyService { + + Logger log = LoggerFactory.getLogger(StockCompanyService.class); + @Autowired + private TushareSpider tushareSpider; + @Autowired + private MongoTemplate mongoTemplate; + + /** + * 代码列表刷新 + */ + public void refreshStockCompany(){ + List list = new ArrayList(); + JSONArray rows_sh =tushareSpider.getStockShCompany(); + JSONArray rows_sz =tushareSpider.getStockSZCompany(); + for (int i = 0; i < rows_sh.size(); i++) { + list.add(new StockCompany(rows_sh.getJSONArray(i))); + } + for (int i = 0; i < rows_sz.size(); i++) { + list.add(new StockCompany(rows_sz.getJSONArray(i))); + } + //先删除再新增 + mongoTemplate.dropCollection(StockCompany.class); + mongoTemplate.insertAll(list); + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockInfoService.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockInfoService.java new file mode 100644 index 00000000..187c6c30 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockInfoService.java @@ -0,0 +1,86 @@ +package io.github.kingschan1204.istock.module.maindata.services; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import io.github.kingschan1204.istock.module.maindata.po.StockTopHolders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.AggregationResults; +import org.springframework.data.mongodb.core.aggregation.LookupOperation; +import org.springframework.data.mongodb.core.aggregation.MatchOperation; +import org.springframework.data.mongodb.core.query.BasicQuery; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * + * @author chenguoxiang + * @create 2018-11-02 14:17 + **/ +@Service +public class StockInfoService { + private Logger log = LoggerFactory.getLogger(StockInfoService.class); + @Autowired + private MongoTemplate mongoTemplate; + + /** + * 查公司详细信息 + * @param code + * @return + */ + public JSONObject getStockInfo(String code) { + //设置lookup + LookupOperation lookupOperation = LookupOperation.newLookup().from("stock_code_info").localField("_id").foreignField("_id").as("info"); + LookupOperation lookupOperation1 = LookupOperation.newLookup().from("stock_company").localField("_id").foreignField("_id").as("company"); + //LookupOperation lookupOperation2 = LookupOperation.newLookup().from("stock_top_holders").localField("_id").foreignField("code").as("doc2"); + + //这里分三个部分，先lookup；结果再用project筛选，如果不筛选user里面所有内容，包括密码都会输出；最后sort排序 + Aggregation aggregation = Aggregation.newAggregation( + lookupOperation, lookupOperation1, + //project("id", "demographic", "sizes", "userDoc.realname"), + //Sort(Sort.Direction.ASC, "demographic") + new MatchOperation(Criteria.where("_id").is(code)) + ); + //看具体的查询，有助于理解各个参数的影响 + log.info(aggregation.toString()); + //正式查询 + AggregationResults results = mongoTemplate.aggregate(aggregation, "stock", BasicDBObject.class); + List list = results.getMappedResults(); + JSONArray jsonArray=JSONArray.parseArray(JSON.toJSONString(list)); + JSONObject data =jsonArray.getJSONObject(0); + //查股东 + DBObject dbObject = new BasicDBObject(); + DBObject fieldObject = new BasicDBObject(); + fieldObject.put("_id", false); + fieldObject.put("code", false); + Query query = new BasicQuery(dbObject,fieldObject); + query.addCriteria(Criteria.where("code").is(code)); + List holders= mongoTemplate.find(query, StockTopHolders.class); + /*Map> map =holders.stream().collect(Collectors.groupingBy(StockTopHolders::getHolderName)); + holders=new ArrayList(); + map.forEach((key, value) ->{ + System.out.println(JSON.toJSON(value)); + }); + + System.out.println(map);*/ + JSONArray jsonHolders=JSONArray.parseArray(JSON.toJSONString(holders)); + data.put("holders",jsonHolders); + return data; + + } + + + +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockService.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockService.java index 58afaded..19760c5f 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockService.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockService.java @@ -6,23 +6,29 @@ import com.mongodb.BasicDBObject; import com.mongodb.DBCollection; import com.mongodb.DBObject; +import com.mongodb.WriteResult; +import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; import io.github.kingschan1204.istock.common.util.stock.StockSpider; import io.github.kingschan1204.istock.common.util.stock.impl.JisiluSpilder; import io.github.kingschan1204.istock.module.maindata.po.Stock; import io.github.kingschan1204.istock.module.maindata.po.StockDividend; -import io.github.kingschan1204.istock.module.maindata.po.StockHisDividend; import io.github.kingschan1204.istock.module.maindata.po.StockHisRoe; import io.github.kingschan1204.istock.module.maindata.repository.StockHisDividendRepository; import io.github.kingschan1204.istock.module.maindata.repository.StockRepository; +import io.github.kingschan1204.istock.module.maindata.vo.StockVo; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.mapreduce.MapReduceOptions; +import org.springframework.data.mongodb.core.mapreduce.MapReduceResults; import org.springframework.data.mongodb.core.query.BasicQuery; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Service; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -46,48 +52,19 @@ public class StockService { @Autowired private JisiluSpilder jisiluSpilder; + /** - * add stock code - * - * @param codes - * @throws Exception + * 查询股票 + * @param pageindex 页码 + * @param pagesize 显示多少条 + * @param pcode 代码、名称 + * @param type 市场 + * @param pb 市净值 + * @param dy 股息 + * @param orderfidld 排序字段 + * @param psort 排序方式 + * @return */ - public void addStock(String... codes) throws Exception { - JSONArray jsons = spider.getStockPrice(codes); - for (int i = 0; i < jsons.size(); i++) { - JSONObject json = jsons.getJSONObject(i); - String scode=json.getString("code"); - JSONObject info=spider.getStockInfo(scode); - //"date":"2017-07-07","code":"600519","year":"2016年报","executeDate":"2017-07-01","remark":"10派67.87元(含税)","percent":1.44} - JSONArray dividends=spider.getHistoryDividendRate(scode); - JSONObject dividend; - String date=""; - Double percent=0D; - if(null!=dividends&÷nds.size()>0){ - for (int j = 0; j < dividends.size(); j++) { - if(dividends.getJSONObject(j).getDouble("percent")>0){ - percent=dividends.getJSONObject(j).getDoubleValue("percent"); - date=dividends.getJSONObject(j).getString("date"); - break; - } - } - //save dividend - List stockHisDividendList = JSONArray.parseArray(dividends.toJSONString(),StockHisDividend.class); - template.remove(new Query(Criteria.where("code").is(scode)),StockHisDividend.class); - } - json.put("dividend",percent); - json.put("dividendDate",date); - json.putAll(info); - // his roe - stockHisRoeService.addStockHisRoe(scode); - } - List list = JSON.parseArray(jsons.toJSONString(), Stock.class); - stockRepository.save(list); - } - - - - public String queryStock(int pageindex, int pagesize, final String pcode,final String type,String pb,String dy, String orderfidld, String psort){ DBObject dbObject = new BasicDBObject(); DBObject fieldObject = new BasicDBObject(); @@ -129,7 +106,7 @@ public String queryStock(int pageindex, int pagesize, final String pcode,final S //分页 query.skip((pageindex-1)*pagesize).limit(pagesize); //排序 - List orders = new ArrayList(); //排序 + List orders = new ArrayList(); orders.add(new Sort.Order( "asc".equalsIgnoreCase(psort) ?Sort.Direction.ASC:Sort.Direction.DESC ,orderfidld)); @@ -137,66 +114,22 @@ public String queryStock(int pageindex, int pagesize, final String pcode,final S query.with(sort); //code List list =template.find(query,Stock.class); - JSONArray jsons = JSONArray.parseArray(JSONArray.toJSONString(list)); - JSONObject temp; - for (int i = 0; i temp = JSON.parseArray(JSON.toJSONString(list), StockVo.class); JSONObject data= new JSONObject(); long pagetotal=total%pagesize==0?total/pagesize:total/pagesize+1; - data.put("rows",jsons); - data.put("total",pagetotal);//有多少页 - data.put("records",total);// 总共有多少条记录 + data.put("rows",JSONArray.parseArray(JSON.toJSONString(temp))); + //有多少页 + data.put("total",pagetotal); + // 总共有多少条记录 + data.put("records",total); data.put("page",pageindex); return data.toJSONString(); } /** - * + * 得到股票历史分红信息 * @param code * @return */ @@ -204,7 +137,7 @@ public List getStockDividend(String code){ Query query = new Query(); query.addCriteria(Criteria.where("code").is(code)); //排序 - List orders = new ArrayList(); //排序 + List orders = new ArrayList(); orders.add(new Sort.Order(Sort.Direction.ASC,"title")); Sort sort = new Sort(orders); query.with(sort); @@ -214,11 +147,16 @@ public List getStockDividend(String code){ } + /** + * 得到股票历史roe + * @param code + * @return + */ public List getStockHisRoe(String code){ Query query = new Query(); query.addCriteria(Criteria.where("code").is(code)); //排序 - List orders = new ArrayList(); //排序 + List orders = new ArrayList(); orders.add(new Sort.Order(Sort.Direction.ASC,"year")); Sort sort = new Sort(orders); query.with(sort); @@ -301,4 +239,94 @@ public List crawAndSaveHisPbPe(String code)throws Exception{ return result; } + /** + * 更新股票价格 + * @param codes + * @param spider + * @throws Exception + */ + public void updateStockPrice(List codes,StockSpider spider) throws Exception { + JSONArray jsons = spider.getStockPrice(codes.toArray(new String[]{})); + List stocks = JSON.parseArray(jsons.toJSONString(), Stock.class); + stocks.stream().forEach(stock -> { + template.upsert( + new Query(Criteria.where("_id").is(stock.getCode())), + new Update() + .set("_id", stock.getCode()) + .set("type", stock.getType()) + .set("name", stock.getName()) + .set("price", stock.getPrice()) + .set("yesterdayPrice", stock.getYesterdayPrice()) + .set("fluctuate", stock.getFluctuate()) + .set("todayMax", stock.getTodayMax()) + .set("todayMin", stock.getTodayMin()) + .set("priceDate", stock.getPriceDate()), + "stock" + ); + }); + } + + /** + * 计算过去5年连续分红的平均股票股息 + * @return + */ + public void calculateFiveYearsDy() { + int year = StockDateUtil.getCurrentYear(); + int fiveYearAgo = year - 5; + String startDate = StockDateUtil.getCurrYearLastDay(fiveYearAgo).toString(); + String endDate = StockDateUtil.getCurrentDate(); + Query query = new Query(); + query.addCriteria(Criteria.where("releaseDate").gte(startDate).lte(endDate)); + MapReduceResults result = template.mapReduce(query, "stock_dividend", + "classpath:/mapreduce/5years_dy/dy5years_map.js", "classpath:/mapreduce/5years_dy/dy5years_reduce.js", + new MapReduceOptions().outputCollection("stock_dy_statistics"), BasicDBObject.class); + Iterator iter = result.iterator(); + while (iter.hasNext()) { + BasicDBObject item = iter.next(); + String code = item.getString("_id"); + BasicDBObject value = (BasicDBObject) item.get("value"); + if (value.containsField("size") && value.getInt("size") > 4) { + double percent = Double.parseDouble(value.getString("percent")); + WriteResult wr = template.upsert( + new Query(Criteria.where("_id").is(code)), + new Update() + .set("_id", code) + .set("fiveYearDy", percent), + "stock" + ); + } + + } + } + + /** + * 计算上市超5年的股息roe + * @param startYear 开始年份 + * @param endYear 结束年份 + */ + public void calculateFiveYearsRoe(int startYear,int endYear) { + Query query = new Query(); + query.addCriteria(Criteria.where("year").gte(startYear).lte(endYear)); + MapReduceResults result = template.mapReduce(query, "stock_his_roe", + "classpath:/mapreduce/5years_roe/map.js", "classpath:/mapreduce/5years_roe/reduce.js", + new MapReduceOptions().outputCollection("stock_hisroe_statistics"), BasicDBObject.class); + Iterator iter = result.iterator(); + while (iter.hasNext()) { + BasicDBObject item = iter.next(); + String code = item.getString("_id"); + BasicDBObject value = (BasicDBObject) item.get("value"); + if (value.containsField("size") && value.getInt("size") > 4) { + double percent = Double.parseDouble(value.getString("percent")); + WriteResult wr = template.upsert( + new Query(Criteria.where("_id").is(code)), + new Update() + .set("_id", code) + .set("fiveYearRoe", percent), + "stock" + ); + } + + } + } + } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockTopHoldersService.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockTopHoldersService.java new file mode 100644 index 00000000..8c68ad49 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockTopHoldersService.java @@ -0,0 +1,56 @@ +package io.github.kingschan1204.istock.module.maindata.services; + +import com.alibaba.fastjson.JSONArray; +import com.mongodb.WriteResult; +import io.github.kingschan1204.istock.common.util.stock.impl.TushareSpider; +import io.github.kingschan1204.istock.module.maindata.po.StockTopHolders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +/** + * 代码前几名持有人 + * @author chenguoxiang + * @create 2018-11-01 10:55 + **/ +@Service +public class StockTopHoldersService { + private Logger log = LoggerFactory.getLogger(StockTopHoldersService.class); + @Autowired + private MongoTemplate mongoTemplate; + @Autowired + private TushareSpider tushareSpider; + + /** + * 刷新持有人 + * @param code + */ + public void refreshTopHolders(String code){ + List list = new ArrayList(); + JSONArray rows =tushareSpider.getStockTopHolders(code); + String frist=null; + for (int i = 0; i < rows.size(); i++) { + if(null==frist){ + frist=rows.getJSONArray(i).getString(1); + } + if(frist.equals(rows.getJSONArray(i).getString(1))){ + list.add(new StockTopHolders(rows.getJSONArray(i))); + }else{ + break; + } + } + String query_code=code.replaceAll("\\D+",""); + //先删除再新增 + WriteResult wr= mongoTemplate.remove(new Query(Criteria.where("code").is(query_code)),StockTopHolders.class); + mongoTemplate.insertAll(list); + log.info("top10 holders :remove {} ,insert {} ",wr.getN(),list.size()); + } + +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/vo/StockVo.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/vo/StockVo.java new file mode 100644 index 00000000..a3af20d2 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/vo/StockVo.java @@ -0,0 +1,228 @@ +package io.github.kingschan1204.istock.module.maindata.vo; + +/** + * stock 主数据vo + * @author chenguoxiang + * @create 2018-10-16 14:41 + * http://localhost/stock/q?rows=20&page=1&sidx=&sord=asc& + **/ +public class StockVo { + + private String code; + private String type; + private String name; + private Double price; + private Double yesterdayPrice; + private String fluctuate; + private Double todayMax; + private Double todayMin; + private Long priceDate; + private String industry; + private String totalValue; + private String pb; + private String roe; + private String bvps; + private String pes; + private String ped; + private Integer infoDate; + /** + * 静态分红日期 + */ + private String dividendDate; + private String dividend; + /** + * 实时股息率 + */ + private String dy; + /** + * 5年平均股息 + */ + private String fiveYearDy; + /** + * 5年平均Roe + */ + private String fiveYearRoe; + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Double getPrice() { + return price; + } + + public void setPrice(Double price) { + this.price = price; + } + + public Double getYesterdayPrice() { + return yesterdayPrice; + } + + public void setYesterdayPrice(Double yesterdayPrice) { + this.yesterdayPrice = yesterdayPrice; + } + + public String getFluctuate() { + return fluctuate; + } + + public void setFluctuate(Double fluctuate) { + this.fluctuate = String.format("%s%%",fluctuate); + } + + public Double getTodayMax() { + return todayMax; + } + + public void setTodayMax(Double todayMax) { + this.todayMax = todayMax; + } + + public Double getTodayMin() { + return todayMin; + } + + public void setTodayMin(Double todayMin) { + this.todayMin = todayMin; + } + + public Long getPriceDate() { + return priceDate; + } + + public void setPriceDate(Long priceDate) { + this.priceDate = priceDate; + } + + public String getIndustry() { + return industry; + } + + public void setIndustry(String industry) { + this.industry = industry; + } + + public String getTotalValue() { + return totalValue; + } + + public void setTotalValue(Double totalValue) { + this.totalValue = -1==totalValue?"--":String.format("%s亿",totalValue); + } + + public String getPb() { + return pb; + } + + public void setPb(Double pb) { + this.pb = -1==pb?"--":String.format("%s",pb); + } + + public String getRoe() { + return roe; + } + + public void setRoe(Double roe) { + this.roe = -1==roe?"--":String.format("%s%%",roe); + } + + public String getBvps() { + return bvps; + } + + public void setBvps(Double bvps) { + this.bvps = -1==bvps?"--":String.format("%s",bvps); + } + + public String getPes() { + return pes; + } + + public void setPes(Double pes) { + this.pes = -1==pes?"--":String.format("%s",pes); + } + + public String getPed() { + return ped; + } + + public void setPed(Double ped) { + this.ped = -1==ped?"--":String.format("%s",ped); + } + + public Integer getInfoDate() { + return infoDate; + } + + public void setInfoDate(Integer infoDate) { + this.infoDate = infoDate; + } + + public String getDividendDate() { + return dividendDate; + } + + public void setDividendDate(String dividendDate) { + this.dividendDate = dividendDate; + } + + public String getDividend() { + return dividend; + } + + public void setDividend(Double dividend) { + if(0==dividend){ + this.dividend=""; + return; + } + this.dividend = -1==dividend?"--":String.format("%s%%",dividend); + } + + public String getDy() { + return dy; + } + + public void setDy(Double dy) { + this.dy = -1==dy?"--":String.format("%s%%",dy); + } + + public String getFiveYearDy() { + return fiveYearDy; + } + + public void setFiveYearDy(Double fiveYearDy) { + this.fiveYearDy = -1==fiveYearDy?"--":String.format("%s%%",fiveYearDy); + } + + public String getFiveYearRoe() { + return fiveYearRoe; + } + + public void setFiveYearRoe(Double fiveYearRoe) { + this.fiveYearRoe = -1==fiveYearRoe?"--":String.format("%s%%",fiveYearRoe); + } + + public StockVo() { + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/startup/InitQuartzTaskRunner.java b/src/main/java/io.github.kingschan1204.istock/module/startup/InitQuartzTaskRunner.java index 0180195b..acf7fedd 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/startup/InitQuartzTaskRunner.java +++ b/src/main/java/io.github.kingschan1204.istock/module/startup/InitQuartzTaskRunner.java @@ -24,7 +24,28 @@ public void run(ApplicationArguments applicationArguments) throws Exception { "sinaPriceTask-trigger", "sinaPriceTask-trigger-group", SinaStockPriceTask.class, - "0 0/2 * * * ?"); + "0/30 * * * * ?"); + + quartzManager.addJob("tencentPriceTask", + "tencentPriceTask-group", + "tencentPriceTask-trigger", + "tencentPriceTask-trigger-group", + TencentStockPriceTask.class, + "0/30 * * * * ?"); + + quartzManager.addJob("stockInfoTask", + "stockInfoTask-group", + "stockInfoTask-trigger", + "stockInfoTask-trigger-group", + ThsStockInfoTask.class, + "0/30 * * * * ?"); + + quartzManager.addJob("stockHoldersTask", + "stockHoldersTask-group", + "stockHoldersTask-trigger", + "stockHoldersTask-trigger-group", + StockTopHoldersTask.class, + "0/30 * * * * ?"); quartzManager.addJob("stockCodeTask", "stockCodeTask-group", @@ -38,7 +59,7 @@ public void run(ApplicationArguments applicationArguments) throws Exception { "stockDividendTask-trigger", "stockDividendTask-trigger-group", StockDividendTask.class, - "*/6 * * * * ?"); + "6 * * * * ?"); quartzManager.addJob("hisRepoartTask", @@ -46,14 +67,9 @@ public void run(ApplicationArguments applicationArguments) throws Exception { "hisRepoartTask-trigger", "hisRepoartTask-trigger-group", ThsHisYearReportTask.class, - "*/6 * * * * ?"); + "6 * * * * ?"); + - quartzManager.addJob("stockInfoTask", - "stockInfoTask-group", - "stockInfoTask-trigger", - "stockInfoTask-trigger-group", - ThsStockInfoTask.class, - "*/6 * * * * ?"); quartzManager.addJob("xueqiuDyTask", "xueqiuDyTask-group", @@ -61,6 +77,13 @@ public void run(ApplicationArguments applicationArguments) throws Exception { "xueqiuDyTask-trigger-group", XueQiuStockDyTask.class, "0 0/1 * * * ?"); + + quartzManager.addJob("CleanFileTask", + "CleanFileTask-group", + "CleanFileTask-trigger", + "CleanFileTask-trigger-group", + CleanFileTask.class, + "0 0 0 * * ?"); } @Override diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/CleanFileTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/CleanFileTask.java new file mode 100644 index 00000000..abbd4bea --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/task/CleanFileTask.java @@ -0,0 +1,40 @@ +package io.github.kingschan1204.istock.module.task; + +import io.github.kingschan1204.istock.module.maindata.po.StockHisPbPe; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Component; +import java.io.File; +import java.util.Arrays; + +/** + * 数据文件清理 + * @author chenguoxiang + * @create 2018-10-10 14:51 + **/ +@Component +public class CleanFileTask implements Job { + + private static Logger log = LoggerFactory.getLogger(CleanFileTask.class); + @Autowired + private MongoTemplate mongoTemplate; + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + + File f = new File("./data/"); + if(!f.exists()){return;} + Arrays.stream(f.listFiles()).forEach(file ->{ + log.info("清理文件:{} {}", file.getName(),file.delete()); + }); + + mongoTemplate.dropCollection(StockHisPbPe.class); + log.info("{}","删除历史pb,pe,price数据"); + + } + +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/SinaStockPriceTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/SinaStockPriceTask.java index 239663c7..0afb982a 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/SinaStockPriceTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/SinaStockPriceTask.java @@ -1,12 +1,10 @@ package io.github.kingschan1204.istock.module.task; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; import io.github.kingschan1204.istock.common.util.stock.StockSpider; -import io.github.kingschan1204.istock.module.maindata.po.Stock; -import io.github.kingschan1204.istock.module.maindata.po.StockCode; -import io.github.kingschan1204.istock.module.maindata.services.StockCodeService; +import io.github.kingschan1204.istock.module.maindata.po.StockCodeInfo; +import io.github.kingschan1204.istock.module.maindata.services.StockCodeInfoService; +import io.github.kingschan1204.istock.module.maindata.services.StockService; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -14,17 +12,12 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.mongodb.core.query.Update; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; - import java.util.ArrayList; import java.util.List; /** - * 定时更新股票价格 + * 定时更新深市股票价格 * * @author chenguoxiang * @create 2018-03-29 14:50 @@ -39,11 +32,9 @@ public class SinaStockPriceTask implements Job { @Autowired private MongoTemplate template; @Autowired - private StockCodeService stockCodeService; - -// @Scheduled(cron = "0 0/2 * * * ?") - /* public void stockPriceExecute() throws Exception { - }*/ + private StockCodeInfoService stockCodeInfoService; + @Autowired + private StockService stockService; @Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { @@ -51,15 +42,14 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution return; } Long start = System.currentTimeMillis(); - List codes = stockCodeService.getAllStockCodes(); + List codes = stockCodeInfoService.getSZStockCodes(); List list = new ArrayList<>(); for (int i = 0; i < codes.size(); i++) { - list.add(codes.get(i).getCode()); + list.add(String.format("%s%s",codes.get(i).getType(),codes.get(i).getCode())); if (i > 0 && (i % 300 == 0 || i == codes.size() - 1)) { try { - updateStockPrice(list); + stockService.updateStockPrice(list, spider); list = new ArrayList<>(); - log.info(String.format("stock更新耗时：%s ms", (System.currentTimeMillis() - start))); Thread.sleep(800); } catch (Exception ex) { log.error("{}", ex); @@ -68,27 +58,8 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution } } - } + log.info("深市数据更新共：{}只股票,更新耗时：{}ms", codes.size(), (System.currentTimeMillis() - start)); - public void updateStockPrice(List codes) throws Exception { - JSONArray jsons = spider.getStockPrice(codes.toArray(new String[]{})); - List stocks = JSON.parseArray(jsons.toJSONString(), Stock.class); - stocks.stream().forEach(stock -> { - template.upsert( - new Query(Criteria.where("_id").is(stock.getCode())), - new Update() - .set("_id", stock.getCode()) - .set("type", stock.getType()) - .set("name", stock.getName()) - .set("price", stock.getPrice()) - .set("yesterdayPrice", stock.getYesterdayPrice()) - .set("fluctuate", stock.getFluctuate()) - .set("todayMax", stock.getTodayMax()) - .set("todayMin", stock.getTodayMin()) - .set("priceDate", stock.getPriceDate()), - "stock" - ); - }); } diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/StockCodeTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/StockCodeTask.java index ee5d94e2..d9401d22 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/StockCodeTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/StockCodeTask.java @@ -1,7 +1,8 @@ package io.github.kingschan1204.istock.module.task; import io.github.kingschan1204.istock.common.util.stock.StockSpider; -import io.github.kingschan1204.istock.module.maindata.services.StockCodeService; +import io.github.kingschan1204.istock.module.maindata.services.StockCodeInfoService; +import io.github.kingschan1204.istock.module.maindata.services.StockCompanyService; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -9,11 +10,11 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; /** - * + * 代码定时更新任务 + * @author kings.chan */ @Component public class StockCodeTask implements Job{ @@ -25,18 +26,16 @@ public class StockCodeTask implements Job{ @Autowired private MongoTemplate template; @Autowired - private StockCodeService stockCodeService; - -/*// @Scheduled(cron = "0 0 0 * * ?") - public void stockCodeExecute()throws Exception{ - - }*/ + private StockCodeInfoService stockCodeInfoService; + @Autowired + private StockCompanyService stockCompanyService; @Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { Long start =System.currentTimeMillis(); try { - stockCodeService.saveAllStockCode(); + stockCodeInfoService.refreshCode(); + stockCompanyService.refreshStockCompany(); } catch (Exception e) { log.error("代码更新错误：{}",e); e.printStackTrace(); diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/StockDividendTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/StockDividendTask.java index f155a6cf..d630730f 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/StockDividendTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/StockDividendTask.java @@ -18,11 +18,11 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; - import javax.annotation.Resource; -import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Set; /** * 定时更新分红情况 @@ -44,32 +44,27 @@ public class StockDividendTask implements Job { @Autowired private StockHisDividendRepository stockHisDividendRepository; -// @Scheduled(cron = "*/6 * * * * ?") - /* public void stockDividendExecute() throws Exception { - - - }*/ - /** - * 同花顺和东方财富的并集 + * 同花顺和东方财富的并集 + * * @param code * @return * @throws Exception */ - public JSONArray combineHisDy(String code)throws Exception{ - JSONArray ths =defaultSpider.getHistoryDividendRate(code); - JSONArray east =eastmoneySpider.getHistoryDividendRate(code); - if(null==east){ - east=new JSONArray(); + public JSONArray combineHisDy(String code) throws Exception { + JSONArray ths = defaultSpider.getHistoryDividendRate(code); + JSONArray east = eastmoneySpider.getHistoryDividendRate(code); + if (null == east) { + east = new JSONArray(); } - if(null!=ths){ + if (null != ths) { east.addAll(ths); } - JSONArray ret=new JSONArray(); - Set titles=new HashSet(); + JSONArray ret = new JSONArray(); + Set titles = new HashSet(); for (int i = 0; i < east.size(); i++) { - String title =east.getJSONObject(i).getString("title"); - if(!titles.contains(title)){ + String title = east.getJSONObject(i).getString("title"); + if (!titles.contains(title)) { ret.add(east.getJSONObject(i)); titles.add(title); } @@ -79,53 +74,56 @@ public JSONArray combineHisDy(String code)throws Exception{ @Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { - Integer dateNumber = StockDateUtil.getCurrentDateNumber()-3; - Criteria cr = new Criteria(); - Criteria c1 = Criteria.where("dividendUpdateDay").lt(dateNumber);//小于 （3天更新一遍） - Criteria c2 = Criteria.where("dividendUpdateDay").exists(false); - Query query = new Query(cr.orOperator(c1, c2)); - query.limit(3); - List list = template.find(query, Stock.class); - for (Stock stock:list) { - Long start =System.currentTimeMillis();//记录开始时间 - int affected=0;//分红更新记录条数 - JSONArray dividends=new JSONArray(); - String date=""; - Double percent=0D; - try { - dividends=combineHisDy(stock.getCode()); - if(null!=dividends&÷nds.size()>0){ - for (int j = 0; j < dividends.size(); j++) { - double tempPercent=dividends.getJSONObject(j).getDouble("percent"); - String tempDate=dividends.getJSONObject(j).getString("cxcqr"); - if(tempPercent>=0&&!"-".equals(tempDate)){ - percent=tempPercent; - date=tempDate; - break; - } + Integer dateNumber = StockDateUtil.getCurrentDateNumber() - 3; + Criteria cr = new Criteria(); + //小于 （3天更新一遍） + Criteria c1 = Criteria.where("dividendUpdateDay").lt(dateNumber); + Criteria c2 = Criteria.where("dividendUpdateDay").exists(false); + Query query = new Query(cr.orOperator(c1, c2)); + query.limit(3); + List list = template.find(query, Stock.class); + for (Stock stock : list) { + //记录开始时间 + Long start = System.currentTimeMillis(); + //分红更新记录条数 + int affected = 0; + JSONArray dividends = new JSONArray(); + String date = ""; + Double percent = 0D; + try { + dividends = combineHisDy(stock.getCode()); + if (null != dividends && dividends.size() > 0) { + for (int j = 0; j < dividends.size(); j++) { + double tempPercent = dividends.getJSONObject(j).getDouble("percent"); + String tempDate = dividends.getJSONObject(j).getString("cxcqr"); + if (tempPercent >= 0 && !"-".equals(tempDate)) { + percent = tempPercent; + date = tempDate; + break; } - //save dividend - List stockDividendList = JSONArray.parseArray(dividends.toJSONString(),StockDividend.class); - template.remove(new Query(Criteria.where("code").is(stock.getCode())),StockDividend.class); - stockHisDividendRepository.save(stockDividendList); - affected=stockDividendList.size(); } - } catch (Exception e) { - log.error("error:{}",e); - e.printStackTrace(); + //save dividend + List stockDividendList = JSONArray.parseArray(dividends.toJSONString(), StockDividend.class); + template.remove(new Query(Criteria.where("code").is(stock.getCode())), StockDividend.class); + stockHisDividendRepository.save(stockDividendList); + affected = stockDividendList.size(); } - WriteResult wr =template.upsert( - new Query(Criteria.where("_id").is(stock.getCode())), - new Update() - .set("_id", stock.getCode()) - .set("dividendDate", date) - .set("dividend", percent) - .set("dividendUpdateDay", dateNumber), - "stock" - ); - affected+=wr.getN(); - log.info("{}分红抓取,耗时{},获得{}行数据",stock.getCode(),(System.currentTimeMillis()-start),affected); + } catch (Exception e) { + log.error("error:{}", e); + e.printStackTrace(); } + WriteResult wr = template.upsert( + new Query(Criteria.where("_id").is(stock.getCode())), + new Update() + .set("_id", stock.getCode()) + .set("dividendDate", date) + .set("dividend", percent) + .set("dividendUpdateDay", dateNumber), + "stock" + ); + affected += wr.getN(); + log.info("{}分红抓取,耗时{}ms,获得{}行数据", stock.getCode(), (System.currentTimeMillis() - start), affected); + } } } diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/StockTopHoldersTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/StockTopHoldersTask.java new file mode 100644 index 00000000..69d6d03c --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/task/StockTopHoldersTask.java @@ -0,0 +1,78 @@ +package io.github.kingschan1204.istock.module.task; + +import com.alibaba.fastjson.JSONObject; +import com.mongodb.WriteResult; +import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; +import io.github.kingschan1204.istock.common.util.stock.StockSpider; +import io.github.kingschan1204.istock.common.util.stock.impl.TushareSpider; +import io.github.kingschan1204.istock.module.maindata.po.Stock; +import io.github.kingschan1204.istock.module.maindata.po.StockCodeInfo; +import io.github.kingschan1204.istock.module.maindata.repository.StockHisDividendRepository; +import io.github.kingschan1204.istock.module.maindata.services.StockTopHoldersService; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 定时更新前10名持有人信息 + * + * @author chenguoxiang + * @create 2018-11-1 14:50 + **/ +@Component +public class StockTopHoldersTask implements Job{ + + private Logger log = LoggerFactory.getLogger(StockTopHoldersTask.class); + + @Autowired + private StockSpider spider; + @Autowired + private MongoTemplate template; + @Autowired + private StockTopHoldersService stockTopHoldersService; + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + /*try { + if (!StockSpider.isWorkDay(StockDateUtil.getCurrentDateNumber())) { + return; + } + } catch (IOException e) { + e.printStackTrace(); + }*/ + Long start = System.currentTimeMillis(); + Integer dateNumber = StockDateUtil.getCurrentDateNumber(); + Criteria cr = new Criteria(); + Criteria c1 = Criteria.where("holdersDate").lt(dateNumber); + Criteria c2 = Criteria.where("holdersDate").exists(false); + Query query = new Query(cr.orOperator(c1,c2)); + query.limit(4); + List list = template.find(query, StockCodeInfo.class); + if(null==list||list.size()==0){ + return ; + } + for (StockCodeInfo stock :list) { + try { + stockTopHoldersService.refreshTopHolders(TushareSpider.formatCode(stock.getCode())); + template.upsert( + new Query(Criteria.where("code").is(stock.getCode())), + new Update().set("holdersDate",StockDateUtil.getCurrentDateNumber()),"stock_code_info"); + } catch (Exception e) { + e.printStackTrace(); + log.error("{}",e); + ; + } + } + log.info(String.format("update stock top 10 holders use ：%s ms ", (System.currentTimeMillis() - start))); + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/TencentStockPriceTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/TencentStockPriceTask.java new file mode 100644 index 00000000..7fe278b8 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/task/TencentStockPriceTask.java @@ -0,0 +1,69 @@ +package io.github.kingschan1204.istock.module.task; + +import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; +import io.github.kingschan1204.istock.common.util.stock.StockSpider; +import io.github.kingschan1204.istock.module.maindata.po.StockCodeInfo; +import io.github.kingschan1204.istock.module.maindata.services.StockCodeInfoService; +import io.github.kingschan1204.istock.module.maindata.services.StockService; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Component; +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * 定时更新沪市股票价格 + * + * @author chenguoxiang + * @create 2018-10-24 14:50 + **/ +@Component +public class TencentStockPriceTask implements Job { + + private Logger log = LoggerFactory.getLogger(TencentStockPriceTask.class); + + @Resource(name = "TencentSpider") + private StockSpider spider; + @Autowired + private MongoTemplate template; + @Autowired + private StockCodeInfoService stockCodeInfoService; + @Autowired + private StockService stockService; + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + if (!StockDateUtil.stockOpenTime()) { + return; + } + Long start = System.currentTimeMillis(); + List codes = stockCodeInfoService.getSHStockCodes(); + List list = new ArrayList<>(); + for (int i = 0; i < codes.size(); i++) { + list.add(String.format("%s%s",codes.get(i).getType(),codes.get(i).getCode())); + if (i > 0 && (i % 300 == 0 || i == codes.size() - 1)) { + try { + stockService.updateStockPrice(list,spider); + list = new ArrayList<>(); + TimeUnit.MILLISECONDS.sleep(800); + } catch (Exception ex) { + log.error("{}", ex); + ex.printStackTrace(); + } + } + + } + log.info("沪市数据更新共：{}只股票,更新耗时：{}ms",codes.size(),(System.currentTimeMillis() - start)); + } + + + + +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/ThsHisYearReportTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/ThsHisYearReportTask.java index 393bd131..8feaf254 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/ThsHisYearReportTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/ThsHisYearReportTask.java @@ -5,7 +5,7 @@ import io.github.kingschan1204.istock.common.util.cache.EhcacheUtil; import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; import io.github.kingschan1204.istock.common.util.stock.StockSpider; -import io.github.kingschan1204.istock.module.maindata.po.StockCode; +import io.github.kingschan1204.istock.module.maindata.po.StockCodeInfo; import io.github.kingschan1204.istock.module.maindata.po.StockHisRoe; import io.github.kingschan1204.istock.module.maindata.repository.StockHisRoeRepository; import org.quartz.Job; @@ -14,14 +14,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import java.util.ArrayList; + import java.util.List; /** @@ -62,10 +60,6 @@ int getErrorTotal(){ } -// @Scheduled(cron = "*/6 * * * * ?") - /* public void execute() throws Exception { - }*/ - @Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { Long start = System.currentTimeMillis(); @@ -76,13 +70,13 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution Integer dateNumber = StockDateUtil.getCurrentDateNumber(); Criteria cr = new Criteria(); //3天更新一把 - Criteria c1 = Criteria.where("hrdud").lt(dateNumber-3); + Criteria c1 = Criteria.where("yearReportDate").lt(dateNumber-3); Criteria c2 = Criteria.where("xlsError").is(0); - Query query = new Query(cr.andOperator(c1,c2)); + Criteria c3 = Criteria.where("yearReportDate").exists(false); + Query query = new Query(cr.orOperator(c3,new Criteria().andOperator(c1,c2))); query.limit(2); - List list = template.find(query, StockCode.class); + List list = template.find(query, StockCodeInfo.class); if(null==list||list.size()==0){ - log.info("his year report data 当前已全部更新完!"); return ; } list.stream().forEach(code ->{ @@ -95,15 +89,15 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution stockHisRoeRepository.save(lis); template.upsert( new Query(Criteria.where("_id").is(code.getCode())), - new Update().set("hrdud", dateNumber), - "stock_code" + new Update().set("yearReportDate", dateNumber), + "stock_code_info" ); } catch (Exception e) { e.printStackTrace(); ehcacheUtil.addKey(cacheName,"error",getErrorTotal()+1); template.upsert( new Query(Criteria.where("_id").is(code.getCode())),new Update().set("xlsError", 1), - "stock_code" + "stock_code_info" ); } }); diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/ThsStockInfoTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/ThsStockInfoTask.java index eb96ca37..14da6e3d 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/ThsStockInfoTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/ThsStockInfoTask.java @@ -5,6 +5,7 @@ import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; import io.github.kingschan1204.istock.common.util.stock.StockSpider; import io.github.kingschan1204.istock.module.maindata.po.Stock; +import io.github.kingschan1204.istock.module.maindata.po.StockCodeInfo; import io.github.kingschan1204.istock.module.maindata.repository.StockHisDividendRepository; import org.quartz.Job; import org.quartz.JobExecutionContext; @@ -19,6 +20,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import java.io.IOException; import java.util.List; /** @@ -39,30 +41,28 @@ public class ThsStockInfoTask implements Job{ @Autowired private StockHisDividendRepository stockHisDividendRepository; -// @Scheduled(cron = "*/6 * * * * ?") - /*public void stockInfoExecute() throws Exception { - - }*/ - @Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { - if (!StockDateUtil.stockOpenTime()) { - return; - } + /*try { + if (!StockSpider.isWorkDay(StockDateUtil.getCurrentDateNumber())) { + return; + } + } catch (IOException e) { + e.printStackTrace(); + }*/ Long start = System.currentTimeMillis(); Integer dateNumber = StockDateUtil.getCurrentDateNumber(); Criteria cr = new Criteria(); Criteria c1 = Criteria.where("infoDate").lt(dateNumber); Criteria c2 = Criteria.where("infoDate").exists(false); Query query = new Query(cr.orOperator(c1,c2)); - query.limit(3); - List list = template.find(query, Stock.class); + query.limit(4); + List list = template.find(query, StockCodeInfo.class); if(null==list||list.size()==0){ - log.info("stock info 今日已全部更新完!"); return ; } int affected=0; - for (Stock stock :list) { + for (StockCodeInfo stock :list) { Stock item = null; try { JSONObject info = spider.getStockInfo(stock.getCode()); @@ -79,10 +79,13 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution .set("roe", item.getRoe()) .set("bvps", item.getBvps()) .set("pes", item.getPes()) - .set("ped", item.getPed()) - .set("infoDate", item.getInfoDate()), + .set("ped", item.getPed()), + //, "stock" ); + template.upsert( + new Query(Criteria.where("_id").is(stock.getCode())), + new Update().set("infoDate", item.getInfoDate()),"stock_code_info"); affected+=wr.getN(); } catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/XueQiuStockDyTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/XueQiuStockDyTask.java index a1997b6a..60da733e 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/XueQiuStockDyTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/XueQiuStockDyTask.java @@ -63,12 +63,10 @@ int getErrorTotal(){ return val; } -// @Scheduled(cron = "0 0/1 * * * ?") - /* public void stockDividendExecute() throws Exception { - }*/ public void uptateDy(JSONObject data) { - int affected = 0;//受影响行 + //受影响行 + int affected = 0; Integer dateNumber = StockDateUtil.getCurrentDateNumber(); JSONArray rows = data.getJSONArray("list"); List list = rows.toJavaList(Stock.class); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 040a655a..ab12f047 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -25,13 +25,14 @@ spider.timeout=8000 spider.useagent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3346.9 Safari/537.36 ## token xueqiu.token= -eastmoney.token=70f12f2f4f091e459a279469fe49eca5 +eastmoney.token= +tushare.token= #security management.context-path=/admin management.security.enabled=false management.security.roles=SUPERUSER -security.user.name=root +security.user.name= security.user.password= ###ehcache spring.cache.ehcache.config=classpath:ehcache.xml diff --git a/src/main/resources/static/plugs.echar/dark.js b/src/main/resources/static/plugs.echar/dark.js new file mode 100644 index 00000000..79ff8c10 --- /dev/null +++ b/src/main/resources/static/plugs.echar/dark.js @@ -0,0 +1,147 @@ +(function (root, factory) { + if (typeof define === 'function' && define.amd) { + // AMD. Register as an anonymous module. + define(['exports', 'echarts'], factory); + } else if (typeof exports === 'object' && typeof exports.nodeName !== 'string') { + // CommonJS + factory(exports, require('echarts')); + } else { + // Browser globals + factory({}, root.echarts); + } +}(this, function (exports, echarts) { + var log = function (msg) { + if (typeof console !== 'undefined') { + console && console.error && console.error(msg); + } + }; + if (!echarts) { + log('ECharts is not Loaded'); + return; + } + var contrastColor = '#eee'; + var axisCommon = function () { + return { + axisLine: { + lineStyle: { + color: contrastColor + } + }, + axisTick: { + lineStyle: { + color: contrastColor + } + }, + axisLabel: { + textStyle: { + color: contrastColor + } + }, + splitLine: { + lineStyle: { + type: 'dashed', + color: '#aaa' + } + }, + splitArea: { + areaStyle: { + color: contrastColor + } + } + }; + }; + + var colorPalette = ['#dd6b66','#759aa0','#e69d87','#8dc1a9','#ea7e53','#eedd78','#73a373','#73b9bc','#7289ab', '#91ca8c','#f49f42']; + var theme = { + color: colorPalette, + backgroundColor: '#333', + tooltip: { + axisPointer: { + lineStyle: { + color: contrastColor + }, + crossStyle: { + color: contrastColor + } + } + }, + legend: { + textStyle: { + color: contrastColor + } + }, + textStyle: { + color: contrastColor + }, + title: { + textStyle: { + color: contrastColor + } + }, + toolbox: { + iconStyle: { + normal: { + borderColor: contrastColor + } + } + }, + dataZoom: { + textStyle: { + color: contrastColor + } + }, + timeline: { + lineStyle: { + color: contrastColor + }, + itemStyle: { + normal: { + color: colorPalette[1] + } + }, + label: { + normal: { + textStyle: { + color: contrastColor + } + } + }, + controlStyle: { + normal: { + color: contrastColor, + borderColor: contrastColor + } + } + }, + timeAxis: axisCommon(), + logAxis: axisCommon(), + valueAxis: axisCommon(), + categoryAxis: axisCommon(), + + line: { + symbol: 'circle' + }, + graph: { + color: colorPalette + }, + gauge: { + title: { + textStyle: { + color: contrastColor + } + } + }, + candlestick: { + itemStyle: { + normal: { + color: '#FD1050', + color0: '#0CF49B', + borderColor: '#FD1050', + borderColor0: '#0CF49B' + } + } + } + }; + theme.categoryAxis.splitLine.show = false; + echarts.registerTheme('dark', theme); +})); \ No newline at end of file diff --git a/src/main/resources/static/plugs/echar/infographic.js b/src/main/resources/static/plugs/echar/infographic.js new file mode 100644 index 00000000..df47b912 --- /dev/null +++ b/src/main/resources/static/plugs/echar/infographic.js @@ -0,0 +1,204 @@ +(function (root, factory) {if (typeof define === 'function' && define.amd) { + // AMD. Register as an anonymous module. + define(['exports', 'echarts'], factory); + } else if (typeof exports === 'object' && typeof exports.nodeName !== 'string') { + // CommonJS + factory(exports, require('echarts')); + } else { + // Browser globals + factory({}, root.echarts); + } +}(this, function (exports, echarts) { + var log = function (msg) { + if (typeof console !== 'undefined') { + console && console.error && console.error(msg); + } + }; + if (!echarts) { + log('ECharts is not Loaded'); + return; + } + + var colorPalette = [ + '#C1232B','#27727B','#FCCE10','#E87C25','#B5C334', + '#FE8463','#9BCA63','#FAD860','#F3A43B','#60C0DD', + '#D7504B','#C6E579','#F4E001','#F0805A','#26C0C0' + ]; + + var theme = { + + color: colorPalette, + + title: { + textStyle: { + fontWeight: 'normal', + color: '#27727B' + } + }, + + visualMap: { + color:['#C1232B','#FCCE10'] + }, + + toolbox: { + iconStyle: { + normal: { + borderColor: colorPalette[0] + } + } + }, + + tooltip: { + backgroundColor: 'rgba(50,50,50,0.5)', + axisPointer : { + type : 'line', + lineStyle : { + color: '#27727B', + type: 'dashed' + }, + crossStyle: { + color: '#27727B' + }, + shadowStyle : { + color: 'rgba(200,200,200,0.3)' + } + } + }, + + dataZoom: { + dataBackgroundColor: 'rgba(181,195,52,0.3)', + fillerColor: 'rgba(181,195,52,0.2)', + handleColor: '#27727B' + }, + + categoryAxis: { + axisLine: { + lineStyle: { + color: '#27727B' + } + }, + splitLine: { + show: false + } + }, + + valueAxis: { + axisLine: { + show: false + }, + splitArea : { + show: false + }, + splitLine: { + lineStyle: { + color: ['#ccc'], + type: 'dashed' + } + } + }, + + timeline: { + lineStyle: { + color: '#27727B' + }, + controlStyle: { + normal: { + color: '#27727B', + borderColor: '#27727B' + } + }, + symbol: 'emptyCircle', + symbolSize: 3 + }, + + line: { + itemStyle: { + normal: { + borderWidth:2, + borderColor:'#fff', + lineStyle: { + width: 3 + } + }, + emphasis: { + borderWidth:0 + } + }, + symbol: 'circle', + symbolSize: 3.5 + }, + + candlestick: { + itemStyle: { + normal: { + color: '#C1232B', + color0: '#B5C334', + lineStyle: { + width: 1, + color: '#C1232B', + color0: '#B5C334' + } + } + } + }, + + graph: { + color: colorPalette + }, + + map: { + label: { + normal: { + textStyle: { + color: '#C1232B' + } + }, + emphasis: { + textStyle: { + color: 'rgb(100,0,0)' + } + } + }, + itemStyle: { + normal: { + areaColor: '#ddd', + borderColor: '#eee' + }, + emphasis: { + areaColor: '#fe994e' + } + } + }, + + gauge: { + axisLine: { + lineStyle: { + color: [[0.2, '#B5C334'],[0.8, '#27727B'],[1, '#C1232B']] + } + }, + axisTick: { + splitNumber: 2, + length: 5, + lineStyle: { + color: '#fff' + } + }, + axisLabel: { + textStyle: { + color: '#fff' + } + }, + splitLine: { + length: '5%', + lineStyle: { + color: '#fff' + } + }, + title : { + offsetCenter: [0, -20] + } + } + }; + + echarts.registerTheme('infographic', theme); +})); \ No newline at end of file diff --git a/src/main/resources/static/plugs/echar/macarons.js b/src/main/resources/static/plugs/echar/macarons.js new file mode 100644 index 00000000..398aa3fd --- /dev/null +++ b/src/main/resources/static/plugs/echar/macarons.js @@ -0,0 +1,198 @@ +(function (root, factory) { + if (typeof define === 'function' && define.amd) { + // AMD. Register as an anonymous module. + define(['exports', 'echarts'], factory); + } else if (typeof exports === 'object' && typeof exports.nodeName !== 'string') { + // CommonJS + factory(exports, require('echarts')); + } else { + // Browser globals + factory({}, root.echarts); + } +}(this, function (exports, echarts) { + var log = function (msg) { + if (typeof console !== 'undefined') { + console && console.error && console.error(msg); + } + }; + if (!echarts) { + log('ECharts is not Loaded'); + return; + } + + var colorPalette = [ + '#2ec7c9','#b6a2de','#5ab1ef','#ffb980','#d87a80', + '#8d98b3','#e5cf0d','#97b552','#95706d','#dc69aa', + '#07a2a4','#9a7fd1','#588dd5','#f5994e','#c05050', + '#59678c','#c9ab00','#7eb00a','#6f5553','#c14089' + ]; + + + var theme = { + color: colorPalette, + + title: { + textStyle: { + fontWeight: 'normal', + color: '#008acd' + } + }, + + visualMap: { + itemWidth: 15, + color: ['#5ab1ef','#e0ffff'] + }, + + toolbox: { + iconStyle: { + normal: { + borderColor: colorPalette[0] + } + } + }, + + tooltip: { + backgroundColor: 'rgba(50,50,50,0.5)', + axisPointer : { + type : 'line', + lineStyle : { + color: '#008acd' + }, + crossStyle: { + color: '#008acd' + }, + shadowStyle : { + color: 'rgba(200,200,200,0.2)' + } + } + }, + + dataZoom: { + dataBackgroundColor: '#efefff', + fillerColor: 'rgba(182,162,222,0.2)', + handleColor: '#008acd' + }, + + grid: { + borderColor: '#eee' + }, + + categoryAxis: { + axisLine: { + lineStyle: { + color: '#008acd' + } + }, + splitLine: { + lineStyle: { + color: ['#eee'] + } + } + }, + + valueAxis: { + axisLine: { + lineStyle: { + color: '#008acd' + } + }, + splitArea : { + show : true, + areaStyle : { + color: ['rgba(250,250,250,0.1)','rgba(200,200,200,0.1)'] + } + }, + splitLine: { + lineStyle: { + color: ['#eee'] + } + } + }, + + timeline : { + lineStyle : { + color : '#008acd' + }, + controlStyle : { + normal : { color : '#008acd'}, + emphasis : { color : '#008acd'} + }, + symbol : 'emptyCircle', + symbolSize : 3 + }, + + line: { + smooth : true, + symbol: 'emptyCircle', + symbolSize: 3 + }, + + candlestick: { + itemStyle: { + normal: { + color: '#d87a80', + color0: '#2ec7c9', + lineStyle: { + color: '#d87a80', + color0: '#2ec7c9' + } + } + } + }, + + scatter: { + symbol: 'circle', + symbolSize: 4 + }, + + map: { + label: { + normal: { + textStyle: { + color: '#d87a80' + } + } + }, + itemStyle: { + normal: { + borderColor: '#eee', + areaColor: '#ddd' + }, + emphasis: { + areaColor: '#fe994e' + } + } + }, + + graph: { + color: colorPalette + }, + + gauge : { + axisLine: { + lineStyle: { + color: [[0.2, '#2ec7c9'],[0.8, '#5ab1ef'],[1, '#d87a80']], + width: 10 + } + }, + axisTick: { + splitNumber: 10, + length :15, + lineStyle: { + color: 'auto' + } + }, + splitLine: { + length :22, + lineStyle: { + color: 'auto' + } + }, + pointer : { + width : 5 + } + } + }; + + echarts.registerTheme('macarons', theme); +})); \ No newline at end of file diff --git a/src/main/resources/templates/index.html b/src/main/resources/templates/index.html index ce03de80..30edec7b 100644 --- a/src/main/resources/templates/index.html +++ b/src/main/resources/templates/index.html @@ -60,7 +60,7 @@ - +