v2.1

er10yi · Dec 27, 2020 · 48a77fd · 48a77fd
1 parent ba80a7c
commit 48a77fd
Show file tree

Hide file tree

Showing 240 changed files with 2,690 additions and 1,896 deletions.
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -1,14 +1,38 @@
 ## ChangeLog
 
-### v2.0.28 - 20201123
-
-1. 前端主机名改成：主机/域名
-2. nmap域名扫描，域名会直接新增到主机/域名中
-3. 任务，[cron任务删除]移到类型列
-4. 修复部分bug，并优化部分前端和后端代码
-5. 移除v2.0.20中[启动任务前，发一次agent心跳包监控，防止agent掉线漏任务]更新
-6. 增加最佳实践文档
-7. 侧边栏增加Logo显示，统一侧边栏菜单图标样式
+### v2.1 - 20201227
+
+#### futures
+1. 增加 nmap局域网扫描结果解析，包括mac地址和设备类型
+2. 增加 检测结果-web信息新增http响应头信息和原始响应，及url链接显示
+3. 检测结果-漏洞 可新增漏洞
+4. 资产管理新增应用系统
+5. 新增分类管理，包含公司部门人员和标签综合分类
+6. 主机/域名，增加应用系统显示，子域名编辑
+7. 资产ip、端口drawer，新增应用系统显示
+8. 新增ip标签，端口标签
+9. 增加企业微信群机器人通知
+10. 增加钉钉群机器人、企微群机器人发送测试消息功能，增加发送测试邮件功能
+11. 增加IM消息，邮件发送日志，可在提醒日志查看（包括异常信息）
+12. 增加高危资产推送，实时推送到企微群和钉钉群
+
+#### optimize
+1. 优化404页面显示
+2. 优化通知设置页面
+3. 优化IM实时通知
+
+#### bug fixes
+1.修复 JavaSambaWeakPass无法使用的问题
+2.修复 域名不规范时，ip端口信息无法入库问题
+3.修复 httpp任务可能无法结束的问题
+
+#### alter
+1. ip项目组相关信息，移动到资产管理
+
+#### remove
+1. 移除插件-代码编辑器
+2. 检测结果，移除url页面
+3. HTTPElasticsearchUnauth插件，去除http服务
 
 ### v2.0.20 - 20201028
 

diff --git a/README.md b/README.md
@@ -12,7 +12,16 @@
 
 魔方-MagiCude，基于Spring Boot微服务架构的系统；具有分布式端口（漏洞）扫描、资产安全管理、实时威胁监控与通知、自动漏洞闭环、漏洞wiki、邮件报告通知、poc框架等功能。
 
-通过魔方-MagiCude可高效获取服务器IP，端口信息，web信息，url链接，漏洞等资产信息，合理的定时扫描任务可伪实时监控端口状态及漏洞状态，并能够对资产进行有效的安全管理；合理建立漏洞wiki，能减少安全部门与业务部门之间的沟通；配置钉钉群机器人后能够即时漏洞提醒，配置邮箱信息后能够定时发送资产报告和漏洞报告；扫描任务会结自动闭环漏洞；通过DNS解析有效获取DHCP的办公机IP与用户关联；具备丰富的图表统计及POC框架功能。
+1. 手动资产管理：应用系统-域名-ip-端口-负责人-标签等信息关联
+2. 自动资产管理：ip、端口(协议、状态、服务、版本)、域内DNS反解析域名、web页面及链接抓取
+3. 高危资产及漏洞实时推送：钉钉群机器人、企微群机器人实时推送
+4. 资产报告和漏洞报告定时推送：邮件汇总报告到默认邮箱、项目负责人只收到自己相关的报告，可配置定时周期
+5. 资产、漏洞统计：折线图、饼图展示相关信息
+6. 漏洞wiki：漏洞原理和修复方案等相关信息，需要持续维护和管理
+7. 基于Python3的自定义插件：方便应急响应
+8. 标签分类：分类可任意添加，可对ip、端口、应用系统手动加标签
+9. 扫描任务管理：实时任务状态，便捷的任务操作
+10. 白名单机制：项目组端口白名单、ip端口扫描白名单、提醒白名单、页面标题白名单、链接域名白名单
 
 ## 简述
 
@@ -22,6 +31,8 @@ agent：端口扫描任务、插件任务、http页面抓取任务执行；从re
 
 端口扫描基于nmap和masscan；主机发现基于nmap ping扫描；安全扫描基于nse脚本和自定义插件，自定义插件包括Java、Python3、基于HTTP/HTTPS访问的json插件，无回显命令执行可使用DNS或HTTP方式辅助确认；web页面抓取基于自定义爬虫。
 
+实时推送：钉钉群机器人、企微群机器人。
+
 ## 安装指南
 
 ### 要求说明

diff --git a/agent/pom.xml b/agent/pom.xml
@@ -8,12 +8,20 @@
     </parent>
     <artifactId>agent</artifactId>
 
+    <properties>
+        <java.version>15</java.version>
+        <!--docker私服地址-->
+        <docker.repository>192.168.12.138:5000</docker.repository>
+        <dockerfile-maven-plugin.version>1.4.13</dockerfile-maven-plugin.version>
+        <jep.version>3.9.1</jep.version>
+        <gson.version>2.8.6</gson.version>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>com.tiji</groupId>
             <artifactId>common</artifactId>
             <version>1.0-SNAPSHOT</version>
-
         </dependency>
         <dependency>
             <groupId>org.springframework.cloud</groupId>
@@ -31,11 +39,11 @@
 
 
         <!-- https://mvnrepository.com/artifact/net.sourceforge.htmlunit/htmlunit -->
-        <!--<dependency>
-            <groupId>net.sourceforge.htmlunit</groupId>
-            <artifactId>htmlunit</artifactId>
-            <version>2.36.0</version>
-        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>net.sourceforge.htmlunit</groupId>-->
+<!--            <artifactId>htmlunit</artifactId>-->
+<!--            <version>2.45.0</version>-->
+<!--        </dependency>-->
         <dependency>
             <groupId>com.tiji</groupId>
             <artifactId>plugin</artifactId>
@@ -47,33 +55,29 @@
         <dependency>
             <groupId>black.ninia</groupId>
             <artifactId>jep</artifactId>
-            <version>3.9.0</version>
+            <version>${jep.version}</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
-            <version>2.8.6</version>
+            <version>${gson.version}</version>
         </dependency>
     </dependencies>
 
-    <properties>
-        <java.version>15</java.version>
-        <!--docker私服地址-->
-        <docker.repository>192.168.12.138:5000</docker.repository>
-    </properties>
+
     <build>
 
         <finalName>agentapp</finalName>
         <!--		本机运行时要去掉resources标签-->
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-                <excludes>
-                    <exclude>application.yml</exclude>
-                </excludes>
-            </resource>
-        </resources>
+<!--        <resources>-->
+<!--            <resource>-->
+<!--                <directory>src/main/resources</directory>-->
+<!--                <excludes>-->
+<!--                    <exclude>application.yml</exclude>-->
+<!--                </excludes>-->
+<!--            </resource>-->
+<!--        </resources>-->
 
         <plugins>
             <plugin>
@@ -94,7 +98,7 @@
 
                 <groupId>com.spotify</groupId>
                 <artifactId>dockerfile-maven-plugin</artifactId>
-                <version>1.4.13</version>
+                <version>${dockerfile-maven-plugin.version}</version>
                 <executions>
                     <execution>
                         <id>default</id>

diff --git a/agent/src/main/java/com/tiji/agent/mq/TaskReceiver.java b/agent/src/main/java/com/tiji/agent/mq/TaskReceiver.java
@@ -281,7 +281,6 @@ public void getMessage(Map<String, String> task) {
                         }
                         break;
                     case "heartbeat":
-                        System.out.println("receive heartbeat...");
                         //InetAddress addr = InetAddress.getLocalHost();
                         //String agentName = "agent_" + addr.getHostName();
                         Map<String, String> agentConfig = new HashMap<>();
@@ -293,7 +292,6 @@ public void getMessage(Map<String, String> task) {
                         //TODO 发送cpu和内存状态
                         if (!agentConfig.isEmpty()) {
                             rabbitMessagingTemplate.convertAndSend("agentconfig", agentConfig);
-                            System.out.println("send to center");
                         }
                         break;
                     default:

diff --git a/agent/src/main/java/com/tiji/agent/thread/ParserThread.java b/agent/src/main/java/com/tiji/agent/thread/ParserThread.java
@@ -59,6 +59,9 @@ public void run() {
                     resultMap.put("bodyWholeText", parserResultMap.get("bodyWholeText"));
                     resultMap.put("scanResult", parserResultMap.get("urlNameAndLink"));
 
+                    resultMap.put("header", parserResultMap.get("header"));
+                    resultMap.put("response", parserResultMap.get("response"));
+
                     if (!currentThread().isInterrupted()) {
                         rabbitMessagingTemplate.convertAndSend("scanresult", resultMap);
 

diff --git a/agent/src/main/java/com/tiji/agent/util/AgentGatherHelper.java b/agent/src/main/java/com/tiji/agent/util/AgentGatherHelper.java
@@ -105,32 +105,21 @@ public X509Certificate[] getAcceptedIssuers() {
         resultMap.put("title", title);
 
         Map<String, String> headersMap = response.headers();
-        headersMap.forEach((key, value) -> {
-            System.out.println(key + " : " + value);
-        });
+        StringBuilder headerBuilder = new StringBuilder();
+        headersMap.forEach((key, value) -> headerBuilder.append(key).append(" : ").append(value).append("\n"));
+        resultMap.put("header", headerBuilder.toString());
+        resultMap.put("response", body.toString());
+
         //响应头
         //Server
         //X-Powered-By
         //Set-Cookie
         //WWW-Authenticate
-        String server = response.header("Server");
-        String x_Powered_By = response.header("X-Powered-By");
-        String set_Cookie = response.header("Set-Cookie");
-        String www_Authenticate = response.header("WWW-Authenticate");
-
-
-        resultMap.put("server", server);
-        resultMap.put("x_Powered_By", x_Powered_By);
-        resultMap.put("set_Cookie", set_Cookie);
-        resultMap.put("www_Authenticate", www_Authenticate);
-
-        //获取全部响应头
-        //StrBuilder reStrBuilder = new StrBuilder();
-        //Map<String, String> responseHeadersMap = response.headers();
-        //for (Map.Entry<String, String> entry : responseHeadersMap.entrySet()) {
-        //    reStrBuilder.append(entry.getKey() + ":" + entry.getValue()).append("\n");
-        //}
-        //resultMap.put("responseServer", reStrBuilder.toString());
+        resultMap.put("server", headersMap.get("Server"));
+        resultMap.put("x_Powered_By", headersMap.get("X-Powered-By"));
+        resultMap.put("set_Cookie", headersMap.get("Set-Cookie"));
+        resultMap.put("www_Authenticate", headersMap.get("WWW-Authenticate"));
+
 
         //TODO title为空，动态js解析一下？
         //感觉没必要....

diff --git a/center/pom.xml b/center/pom.xml
@@ -12,19 +12,31 @@
         <java.version>15</java.version>
         <!--docker私服地址-->
         <docker.repository>192.168.12.138:5000</docker.repository>
+        <dockerfile-maven-plugin.version>1.4.13</dockerfile-maven-plugin.version>
+        <mysql-connector-java.version>8.0.22</mysql-connector-java.version>
+        <commons-lang3.version>3.11</commons-lang3.version>
+        <jjwt.version>0.9.1</jjwt.version>
+        <poi.version>4.1.2</poi.version>
+        <poi-ooxml.version>4.1.2</poi-ooxml.version>
+        <druid-spring-boot-starter.version>1.2.4</druid-spring-boot-starter.version>
+        <devutility.internal.version>1.3.8.1</devutility.internal.version>
+        <httpclient.version>4.5.13</httpclient.version>
+        <gson.version>2.8.6</gson.version>
+        <dnsjava.version>3.2.2</dnsjava.version>
+        <RoaringBitmap.version>0.9.3</RoaringBitmap.version>
     </properties>
 
     <build>
         <finalName>centerapp</finalName>
         <!--		本机运行时要去掉resources标签-->
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-                <excludes>
-                    <exclude>application.yml</exclude>
-                </excludes>
-            </resource>
-        </resources>
+<!--        <resources>-->
+<!--            <resource>-->
+<!--                <directory>src/main/resources</directory>-->
+<!--                <excludes>-->
+<!--                    <exclude>application.yml</exclude>-->
+<!--                </excludes>-->
+<!--            </resource>-->
+<!--        </resources>-->
 
         <plugins>
             <plugin>
@@ -42,7 +54,7 @@
             <plugin>
                 <groupId>com.spotify</groupId>
                 <artifactId>dockerfile-maven-plugin</artifactId>
-                <version>1.4.13</version>
+                <version>${dockerfile-maven-plugin.version}</version>
                 <executions>
                     <execution>
                         <id>default</id>
@@ -92,7 +104,7 @@
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
-            <version>8.0.21</version>
+            <version>${mysql-connector-java.version}</version>
         </dependency>
 
 
@@ -104,7 +116,8 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.11</version>
+            <version>${commons-lang3.version}</version>
+
         </dependency>
         <dependency>
             <groupId>org.springframework.cloud</groupId>
@@ -118,7 +131,7 @@
         <dependency>
             <groupId>io.jsonwebtoken</groupId>
             <artifactId>jjwt</artifactId>
-            <version>0.9.1</version>
+            <version>${jjwt.version}</version>
         </dependency>
 
         <dependency>
@@ -133,13 +146,13 @@
         <dependency>
             <groupId>org.apache.poi</groupId>
             <artifactId>poi</artifactId>
-            <version>4.1.2</version>
+            <version>${poi.version}</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
         <dependency>
             <groupId>org.apache.poi</groupId>
             <artifactId>poi-ooxml</artifactId>
-            <version>4.1.2</version>
+            <version>${poi-ooxml.version}</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -149,30 +162,37 @@
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>druid-spring-boot-starter</artifactId>
-            <version>1.1.22</version>
+            <version>${druid-spring-boot-starter.version}</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/com.github.eagle6688/devutility.internal -->
         <dependency>
             <groupId>com.github.eagle6688</groupId>
             <artifactId>devutility.internal</artifactId>
-            <version>1.3.8.1</version>
+            <version>${devutility.internal.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
+            <version>${httpclient.version}</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
-            <version>2.8.6</version>
+            <version>${gson.version}</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/dnsjava/dnsjava -->
         <dependency>
             <groupId>dnsjava</groupId>
             <artifactId>dnsjava</artifactId>
-            <version>3.2.2</version>
+            <version>${dnsjava.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.roaringbitmap/RoaringBitmap -->
+        <dependency>
+            <groupId>org.roaringbitmap</groupId>
+            <artifactId>RoaringBitmap</artifactId>
+            <version>${RoaringBitmap.version}</version>
         </dependency>
 
     </dependencies>