基础访问监控
# PV统计
网站浏览量，多次打开可以累计；
接口：
public interface LinkAccessStatsMapper extends BaseMapper<LinkAccessStatsDO> {
@Insert("INSERT INTO t_link_access_stats (full_short_url, gid, date, pv, uv, uip, hour, weekday, create_time, update_time, del_flag) " +
"VALUES( #{linkAccessStats.fullShortUrl}, #{linkAccessStats.gid}, #{linkAccessStats.date}, #{linkAccessStats.pv}, #{linkAccessStats.uv}, #{linkAccessStats.uip}, #{linkAccessStats.hour}, #{linkAccessStats.weekday}, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE pv = pv + #{linkAccessStats.pv}, " +
"uv = uv + #{linkAccessStats.uv}, " +
"uip = uip + #{linkAccessStats.uip};")
void shortLinkStats(@Param("linkAccessStats") LinkAccessStatsDO linkAccessStatsDO);
}
1
2
3
4
5
6
7
2
3
4
5
6
7
在restoreUrl方法中调用：
private void shortLinkStats(String fullShortUrl, String gid, ServletRequest request, ServletResponse response) {
try {
if (StrUtil.isBlank(gid)) {
LambdaQueryWrapper<ShortLinkGotoDO> queryWrapper = Wrappers.lambdaQuery(ShortLinkGotoDO.class)
.eq(ShortLinkGotoDO::getFullShortUrl, fullShortUrl);
ShortLinkGotoDO shortLinkGotoDO = shortLinkGotoMapper.selectOne(queryWrapper);
gid = shortLinkGotoDO.getGid();
}
int hour = DateUtil.hour(new Date(), true);
Week week = DateUtil.dayOfWeekEnum(new Date());
int weekValue = week.getIso8601Value();
LinkAccessStatsDO linkAccessStatsDO = LinkAccessStatsDO.builder()
.pv(1)
.uv(1)
.uip(1)
.hour(hour)
.weekday(weekValue)
.fullShortUrl(fullShortUrl)
.gid(gid)
.date(new Date())
.build();
linkAccessStatsMapper.shortLinkStats(linkAccessStatsDO);
} catch (Throwable ex) {
log.error("短链接访问量统计异常", ex);
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# UV统计
独立访客数，一天内同一访客的多次访问只记1个UV，以cookie或token为依据；
当客户端第一次访问某个网站服务器，网站服务器会给这个客户端发一个Cookie，通常放客户端电脑的C盘中，这个cookie中会分配一个独一无二的编号，这其中记录一些访问服务器的信息，如访问时间，访问了哪些页面等等。当你下次再访问这个服务器的时候，服务器就可以直接从你的电脑中找到上一次放进去的Cookie文件，并且对其进行一些更新，但那个独一无二的编号是不会变的；
private void shortLinkStats(String fullShortUrl, String gid, ServletRequest request, ServletResponse response) {
AtomicBoolean uvFirstFlag = new AtomicBoolean();
Cookie[] cookies = ((HttpServletRequest) request).getCookies();
try {
Runnable addResponseCookieTask = () -> {
String uv = UUID.fastUUID().toString();
Cookie uvCookie = new Cookie("uv", uv);
uvCookie.setMaxAge(60 * 60 * 24 * 30);
uvCookie.setPath(StrUtil.sub(fullShortUrl, fullShortUrl.indexOf("/"), fullShortUrl.length()));
((HttpServletResponse) response).addCookie(uvCookie);
uvFirstFlag.set(Boolean.TRUE);
stringRedisTemplate.opsForSet().add("short-link:stats:uv:" + fullShortUrl, uv);
};
if (ArrayUtil.isNotEmpty(cookies)) {
Arrays.stream(cookies)
.filter(each -> Objects.equals(each.getName(), "uv"))
.findFirst()
.map(Cookie::getValue)
.ifPresentOrElse(each -> {
Long added = stringRedisTemplate.opsForSet().add("short-link:stats:uv:" + fullShortUrl, each);
uvFirstFlag.set(added != null && added > 0L);
}, addResponseCookieTask);
} else {
addResponseCookieTask.run();
}
if (StrUtil.isBlank(gid)) {
LambdaQueryWrapper<ShortLinkGotoDO> queryWrapper = Wrappers.lambdaQuery(ShortLinkGotoDO.class)
.eq(ShortLinkGotoDO::getFullShortUrl, fullShortUrl);
ShortLinkGotoDO shortLinkGotoDO = shortLinkGotoMapper.selectOne(queryWrapper);
gid = shortLinkGotoDO.getGid();
}
int hour = DateUtil.hour(new Date(), true);
Week week = DateUtil.dayOfWeekEnum(new Date());
int weekValue = week.getIso8601Value();
LinkAccessStatsDO linkAccessStatsDO = LinkAccessStatsDO.builder()
.pv(1)
.uv(uvFirstFlag.get() ? 1 : 0)
.uip(1)
.hour(hour)
.weekday(weekValue)
.fullShortUrl(fullShortUrl)
.gid(gid)
.date(new Date())
.build();
linkAccessStatsMapper.shortLinkStats(linkAccessStatsDO);
} catch (Throwable ex) {
log.error("短链接访问量统计异常", ex);
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# IP统计
多台机器是同一个IP，也只能算一个IP访问；
private void shortLinkStats(String fullShortUrl, String gid, ServletRequest request, ServletResponse response) {
// 统计UV
AtomicBoolean uvFirstFlag = new AtomicBoolean();
Cookie[] cookies = ((HttpServletRequest) request).getCookies();
try {
Runnable addResponseCookieTask = () -> {
String uv = UUID.fastUUID().toString();
Cookie uvCookie = new Cookie("uv", uv);
uvCookie.setMaxAge(60 * 60 * 24 * 30);
uvCookie.setPath(StrUtil.sub(fullShortUrl, fullShortUrl.indexOf("/"), fullShortUrl.length()));
((HttpServletResponse) response).addCookie(uvCookie);
uvFirstFlag.set(Boolean.TRUE);
stringRedisTemplate.opsForSet().add("short-link:stats:uv:" + fullShortUrl, uv);
};
if (ArrayUtil.isNotEmpty(cookies)) {
Arrays.stream(cookies)
.filter(each -> Objects.equals(each.getName(), "uv"))
.findFirst()
.map(Cookie::getValue)
.ifPresentOrElse(each -> {
Long uvAdded = stringRedisTemplate.opsForSet().add("short-link:stats:uv:" + fullShortUrl, each);
uvFirstFlag.set(uvAdded != null && uvAdded > 0L);
}, addResponseCookieTask);
} else {
addResponseCookieTask.run();
}
// 统计IP
String remoteAddr = LinkUtil.getActualIp(((HttpServletRequest) request));
Long uipAdded = stringRedisTemplate.opsForSet().add("short-link:stats:uip:" + fullShortUrl, remoteAddr);
boolean uipFirstFlag = uipAdded != null && uipAdded > 0L;
// 统计PV
if (StrUtil.isBlank(gid)) {
LambdaQueryWrapper<ShortLinkGotoDO> queryWrapper = Wrappers.lambdaQuery(ShortLinkGotoDO.class)
.eq(ShortLinkGotoDO::getFullShortUrl, fullShortUrl);
ShortLinkGotoDO shortLinkGotoDO = shortLinkGotoMapper.selectOne(queryWrapper);
gid = shortLinkGotoDO.getGid();
}
int hour = DateUtil.hour(new Date(), true);
Week week = DateUtil.dayOfWeekEnum(new Date());
int weekValue = week.getIso8601Value();
LinkAccessStatsDO linkAccessStatsDO = LinkAccessStatsDO.builder()
.pv(1)
.uv(uvFirstFlag.get() ? 1 : 0)
.uip(uipFirstFlag ? 1 : 0)
.hour(hour)
.weekday(weekValue)
.fullShortUrl(fullShortUrl)
.gid(gid)
.date(new Date())
.build();
linkAccessStatsMapper.shortLinkStats(linkAccessStatsDO);
} catch (Throwable ex) {
log.error("短链接访问量统计异常", ex);
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# 地区统计
接口：
public interface LinkLocaleStatsMapper extends BaseMapper<LinkLocaleStatsDO> {
@Insert("INSERT INTO t_link_locale_stats (full_short_url, gid, date, cnt, country, province, city, adcode, create_time, update_time, del_flag) " +
"VALUES( #{linkLocaleStats.fullShortUrl}, #{linkLocaleStats.gid}, #{linkLocaleStats.date}, #{linkLocaleStats.cnt}, #{linkLocaleStats.country}, #{linkLocaleStats.province}, #{linkLocaleStats.city}, #{linkLocaleStats.adcode}, NOW(), NOW(), 0) " +
"ON DUPLICATE KEY UPDATE cnt = cnt + #{linkLocaleStats.cnt};")
void shortLinkLocaleState(@Param("linkLocaleStats") LinkLocaleStatsDO linkLocaleStatsDO);
}
1
2
3
4
5
6
2
3
4
5
6
Map<String, Object> localeParamMap = new HashMap<>();
localeParamMap.put("key", statsLocaleAmapKey);
localeParamMap.put("ip", remoteAddr);
String localeResultStr = HttpUtil.get(AMAP_REMOTE_URL, localeParamMap);
JSONObject localeResultObj = JSON.parseObject(localeResultStr);
String infoCode = localeResultObj.getString("infocode");
if (StrUtil.isNotBlank(infoCode) && StrUtil.equals(infoCode, "10000")) {
String province = localeResultObj.getString("province");
boolean unknownFlag = StrUtil.equals(province, "[]");
LinkLocaleStatsDO linkLocaleStatsDO = LinkLocaleStatsDO.builder()
.province(unknownFlag ? "未知" : province)
.city(unknownFlag ? "未知" : localeResultObj.getString("city"))
.adcode(unknownFlag ? "未知" : localeResultObj.getString("adcode"))
.cnt(1)
.fullShortUrl(fullShortUrl)
.country("中国")
.gid(gid)
.date(new Date())
.build();
linkLocaleStatsMapper.shortLinkLocaleState(linkLocaleStatsDO);
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 操作系统访问统计
接口：
public interface LinkOsStatsMapper extends BaseMapper<LinkOsStatsDO> {
@Insert("INSERT INTO t_link_os_stats (full_short_url, gid, date, cnt, os, create_time, update_time, del_flag) " +
"VALUES( #{linkOsStats.fullShortUrl}, #{linkOsStats.gid}, #{linkOsStats.date}, #{linkOsStats.cnt}, #{linkOsStats.os}, NOW(), NOW(), 0) " +
"ON DUPLICATE KEY UPDATE cnt = cnt + #{linkOsStats.cnt};")
void shortLinkOsState(@Param("linkOsStats") LinkOsStatsDO linkOsStatsDO);
}
1
2
3
4
5
6
2
3
4
5
6
LinkOsStatsDO linkOsStatsDO = LinkOsStatsDO.builder()
.os(LinkUtil.getOs(((HttpServletRequest) request)))
.cnt(1)
.gid(gid)
.fullShortUrl(fullShortUrl)
.date(new Date())
.build();
linkOsStatsMapper.shortLinkOsState(linkOsStatsDO);
1
2
3
4
5
6
7
8
2
3
4
5
6
7
8
# 浏览器统计
接口：
public interface LinkBrowserStatsMapper extends BaseMapper<LinkBrowserStatsDO> {
@Insert("INSERT INTO t_link_browser_stats (full_short_url, gid, date, cnt, browser, create_time, update_time, del_flag) " +
"VALUES( #{linkBrowserStats.fullShortUrl}, #{linkBrowserStats.gid}, #{linkBrowserStats.date}, #{linkBrowserStats.cnt}, #{linkBrowserStats.browser}, NOW(), NOW(), 0) " +
"ON DUPLICATE KEY UPDATE cnt = cnt + #{linkBrowserStats.cnt};")
void shortLinkBrowserState(@Param("linkBrowserStats") LinkBrowserStatsDO linkBrowserStatsDO);
}
1
2
3
4
5
6
2
3
4
5
6
LinkBrowserStatsDO linkBrowserStatsDO = LinkBrowserStatsDO.builder()
.browser(LinkUtil.getBrowser(((HttpServletRequest) request)))
.cnt(1)
.gid(gid)
.fullShortUrl(fullShortUrl)
.date(new Date())
.build();
linkBrowserStatsMapper.shortLinkBrowserState(linkBrowserStatsDO);
1
2
3
4
5
6
7
8
2
3
4
5
6
7
8
# 高频访问IP
AtomicReference 存储 uv
创建日志表；
LinkAccessLogsDO linkAccessLogsDO = LinkAccessLogsDO.builder()
.user(uv.get())
.ip(remoteAddr)
.browser(browser)
.os(os)
.gid(gid)
.fullShortUrl(fullShortUrl)
.build();
linkAccessLogsMapper.insert(linkAccessLogsDO);
1
2
3
4
5
6
7
8
9
2
3
4
5
6
7
8
9
sql 查询：
select ip,COUNT(ip) visit_count
from t_link_access_logs
where full_short_url = ''
group by ip
order by visist_count desc
limit 1;
1
2
3
4
5
6
2
3
4
5
6
# 设备统计
接口：
public interface LinkDeviceStatsMapper extends BaseMapper<LinkDeviceStatsDO> {
@Insert("INSERT INTO t_link_device_stats (full_short_url, gid, date, cnt, device, create_time, update_time, del_flag) " +
"VALUES( #{linkDeviceStats.fullShortUrl}, #{linkDeviceStats.gid}, #{linkDeviceStats.date}, #{linkDeviceStats.cnt}, #{linkDeviceStats.device}, NOW(), NOW(), 0) " +
"ON DUPLICATE KEY UPDATE cnt = cnt + #{linkDeviceStats.cnt};")
void shortLinkDeviceState(@Param("linkDeviceStats") LinkDeviceStatsDO linkDeviceStatsDO);
}
1
2
3
4
5
6
2
3
4
5
6
LinkDeviceStatsDO linkDeviceStatsDO = LinkDeviceStatsDO.builder()
.device(LinkUtil.getDevice(((HttpServletRequest) request)))
.cnt(1)
.gid(gid)
.fullShortUrl(fullShortUrl)
.date(new Date())
.build();
linkDeviceStatsMapper.shortLinkDeviceState(linkDeviceStatsDO);
1
2
3
4
5
6
7
8
2
3
4
5
6
7
8
# 网络统计
接口：
public interface LinkNetworkStatsMapper extends BaseMapper<LinkNetworkStatsDO> {
@Insert("INSERT INTO t_link_network_stats (full_short_url, gid, date, cnt, network, create_time, update_time, del_flag) " +
"VALUES( #{linkNetworkStats.fullShortUrl}, #{linkNetworkStats.gid}, #{linkNetworkStats.date}, #{linkNetworkStats.cnt}, #{linkNetworkStats.network}, NOW(), NOW(), 0) " +
"ON DUPLICATE KEY UPDATE cnt = cnt + #{linkNetworkStats.cnt};")
void shortLinkNetworkState(@Param("linkNetworkStats") LinkNetworkStatsDO linkNetworkStatsDO);
}
1
2
3
4
5
6
2
3
4
5
6
LinkNetworkStatsDO linkNetworkStatsDO = LinkNetworkStatsDO.builder()
.network(LinkUtil.getNetwork(((HttpServletRequest) request)))
.cnt(1)
.gid(gid)
.fullShortUrl(fullShortUrl)
.date(new Date())
.build();
linkNetworkStatsMapper.shortLinkNetworkState(linkNetworkStatsDO);
1
2
3
4
5
6
7
8
2
3
4
5
6
7
8
# 消息队列重构
引入消息队列削峰，防止海量访问短链接，监控数据直接访问数据库，导致数据库负载变高；