幂等性组件
# 整体逻辑
定义一个注解(重要成员变量:uniqueKeyPrefix,scene,type)
定义一个抽象处理器类(使用模板方法,1.构建参数包装类(Wrapper:注解+ProcedingJoinPoint+keyLock),2.幂等处理逻辑);
各种类继承这个抽象类,
MQ 的SpEl场景为例:从Wrapper中获取注解,注解中有一个uniqueKeyPrefix与Wrapper中的锁标识拼接作为uniqueKey;
其它三个方法:
- handler(wrapper):查询缓存与消费状态,如果消费状态为消费中,抛出一个重复消费异常;
- exceptionProcessing():用于删除锁标识;
- psotProcessing():添加锁标识;
定义切面类,重写环绕通知方法,从注解中获取到key,scene,type,再从处理器工厂中获取对应的handler,执行它的excute()方法;
工厂类也是从容器中获取,容器类实现了ApplicationContextAware,再启动的时候就将各种handler的Bean注册到容器中;
切面类逻辑:先执行处理器中的handler方法,增强的方法执行完后(joinPoint.proceed()),执行postProcessing方法;上述三个步骤会捕捉异常,如果是重复消费异常则执行exceptionProcessing()方法;
# 幂等注解
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface Idempotent {
/**
* 幂等Key,只有在 {@link Idempotent#type()} 为 {@link IdempotentTypeEnum#SPEL} 时生效
*/
String key() default "";
/**
* 触发幂等失败逻辑时,返回的错误提示信息
*/
String message() default "您操作太快,请稍后再试";
/**
* 验证幂等类型,支持多种幂等方式
* RestAPI 建议使用 {@link IdempotentTypeEnum#TOKEN} 或 {@link IdempotentTypeEnum#PARAM}
* 其它类型幂等验证,使用 {@link IdempotentTypeEnum#SPEL}
*/
IdempotentTypeEnum type() default IdempotentTypeEnum.PARAM;
/**
* 验证幂等场景,支持多种 {@link IdempotentSceneEnum}
*/
IdempotentSceneEnum scene() default IdempotentSceneEnum.RESTAPI;
/**
* 设置防重令牌 Key 前缀,MQ 幂等去重可选设置
* {@link IdempotentSceneEnum#MQ} and {@link IdempotentTypeEnum#SPEL} 时生效
*/
String uniqueKeyPrefix() default "";
/**
* 设置防重令牌 Key 过期时间,单位秒,默认 1 小时,MQ 幂等去重可选设置
* {@link IdempotentSceneEnum#MQ} and {@link IdempotentTypeEnum#SPEL} 时生效
*/
long keyTimeout() default 3600L;
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# 幂等属性配置类
@Data
@ConfigurationProperties(prefix = IdempotentProperties.PREFIX)
public class IdempotentProperties {
public static final String PREFIX = "idempotent.token";
/**
* Token 幂等 Key 前缀
*/
private String prefix;
/**
* Token 申请后过期时间
* 单位默认毫秒 {@link TimeUnit#MILLISECONDS}
* 随着分布式缓存过期时间单位 {@link RedisDistributedProperties#valueTimeUnit} 而变化
*/
private Long timeout;
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 幂等自动装配类
@EnableConfigurationProperties(IdempotentProperties.class)
public class IdempotentAutoConfiguration {
/**
* 幂等切面
*/
@Bean
public IdempotentAspect idempotentAspect() {
return new IdempotentAspect();
}
/**
* 参数方式幂等实现,基于 RestAPI 场景
*/
@Bean
@ConditionalOnMissingBean
public IdempotentParamService idempotentParamExecuteHandler(RedissonClient redissonClient) {
return new IdempotentParamExecuteHandler(redissonClient);
}
/**
* Token 方式幂等实现,基于 RestAPI 场景
*/
@Bean
@ConditionalOnMissingBean
public IdempotentTokenService idempotentTokenExecuteHandler(DistributedCache distributedCache,
IdempotentProperties idempotentProperties) {
return new IdempotentTokenExecuteHandler(distributedCache, idempotentProperties);
}
/**
* 申请幂等 Token 控制器,基于 RestAPI 场景
*/
@Bean
public IdempotentTokenController idempotentTokenController(IdempotentTokenService idempotentTokenService) {
return new IdempotentTokenController(idempotentTokenService);
}
/**
* SpEL 方式幂等实现,基于 RestAPI 场景
*/
@Bean
@ConditionalOnMissingBean
public IdempotentSpELService idempotentSpELByRestAPIExecuteHandler(RedissonClient redissonClient) {
return new IdempotentSpELByRestAPIExecuteHandler(redissonClient);
}
/**
* SpEL 方式幂等实现,基于 MQ 场景
*/
@Bean
@ConditionalOnMissingBean
public IdempotentSpELByMQExecuteHandler idempotentSpELByMQExecuteHandler(DistributedCache distributedCache) {
return new IdempotentSpELByMQExecuteHandler(distributedCache);
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
# 幂等上下文
key为场景+类型,如:wrapper:spEL:MQ;
val为 IdempotentParamWrapper;
public final class IdempotentContext {
private static final ThreadLocal<Map<String, Object>> CONTEXT = new ThreadLocal<>();
public static Map<String, Object> get() {
return CONTEXT.get();
}
public static Object getKey(String key) {
Map<String, Object> context = get();
if (CollUtil.isNotEmpty(context)) {
return context.get(key);
}
return null;
}
public static String getString(String key) {
Object actual = getKey(key);
if (actual != null) {
return actual.toString();
}
return null;
}
public static void put(String key, Object val) {
Map<String, Object> context = get();
if (CollUtil.isEmpty(context)) {
context = Maps.newHashMap();
}
context.put(key, val);
putContext(context);
}
public static void putContext(Map<String, Object> context) {
Map<String, Object> threadContext = CONTEXT.get();
if (CollUtil.isNotEmpty(threadContext)) {
threadContext.putAll(context);
return;
}
CONTEXT.set(context);
}
public static void clean() {
CONTEXT.remove();
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
# 执行处理器
处理器工厂:
public final class IdempotentExecuteHandlerFactory {
/**
* 获取幂等执行处理器
*
* @param scene 指定幂等验证场景类型
* @param type 指定幂等处理类型
* @return 幂等执行处理器
*/
public static IdempotentExecuteHandler getInstance(IdempotentSceneEnum scene, IdempotentTypeEnum type) {
IdempotentExecuteHandler result = null;
switch (scene) {
case RESTAPI -> {
switch (type) {
case PARAM -> result = ApplicationContextHolder.getBean(IdempotentParamService.class);
case TOKEN -> result = ApplicationContextHolder.getBean(IdempotentTokenService.class);
case SPEL -> result = ApplicationContextHolder.getBean(IdempotentSpELByRestAPIExecuteHandler.class);
default -> {
}
}
}
case MQ -> result = ApplicationContextHolder.getBean(IdempotentSpELByMQExecuteHandler.class);
default -> {
}
}
return result;
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
容器:获取Bean
public class ApplicationContextHolder implements ApplicationContextAware { private static ApplicationContext CONTEXT; @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { ApplicationContextHolder.CONTEXT = applicationContext; } /** * Get ioc container bean by type. */ public static <T> T getBean(Class<T> clazz) { return CONTEXT.getBean(clazz); } //...... }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
public interface IdempotentExecuteHandler {
/**
* 幂等处理逻辑
*
* @param wrapper 幂等参数包装器
*/
void handler(IdempotentParamWrapper wrapper);
/**
* 执行幂等处理逻辑
*
* @param joinPoint AOP 方法处理
* @param idempotent 幂等注解
*/
void execute(ProceedingJoinPoint joinPoint, Idempotent idempotent);
/**
* 异常流程处理
*/
default void exceptionProcessing() {
}
/**
* 后置处理
*/
default void postProcessing() {
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
public abstract class AbstractIdempotentExecuteHandler implements IdempotentExecuteHandler {
/**
* 构建幂等验证过程中所需要的参数包装器
*
* @param joinPoint AOP 方法处理
* @return 幂等参数包装器
*/
protected abstract IdempotentParamWrapper buildWrapper(ProceedingJoinPoint joinPoint);
/**
* 执行幂等处理逻辑
*
* @param joinPoint AOP 方法处理
* @param idempotent 幂等注解
*/
public void execute(ProceedingJoinPoint joinPoint, Idempotent idempotent) {
// 模板方法模式:构建幂等参数包装器
IdempotentParamWrapper idempotentParamWrapper = buildWrapper(joinPoint).setIdempotent(idempotent);
handler(idempotentParamWrapper);
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
@Data @Builder @NoArgsConstructor @AllArgsConstructor @Accessors(chain = true) public final class IdempotentParamWrapper { /** * 幂等注解 */ private Idempotent idempotent; /** * AOP 处理连接点 */ private ProceedingJoinPoint joinPoint; /** * 锁标识,{@link IdempotentTypeEnum#PARAM} */ private String lockKey; }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# SpEL方式幂等
public interface IdempotentSpELService extends IdempotentExecuteHandler {
}
2
# RestAPI场景
请求获取唯一key的锁;
把锁放在幂等上下文中;
@RequiredArgsConstructor
public final class IdempotentSpELByRestAPIExecuteHandler extends AbstractIdempotentExecuteHandler implements IdempotentSpELService {
private final RedissonClient redissonClient;
private final static String LOCK = "lock:spEL:restAPI";
@SneakyThrows
@Override
protected IdempotentParamWrapper buildWrapper(ProceedingJoinPoint joinPoint) {
Idempotent idempotent = IdempotentAspect.getIdempotent(joinPoint);
String key = (String) SpELUtil.parseKey(idempotent.key(), ((MethodSignature) joinPoint.getSignature()).getMethod(), joinPoint.getArgs());
return IdempotentParamWrapper.builder().lockKey(key).joinPoint(joinPoint).build();
}
@Override
public void handler(IdempotentParamWrapper wrapper) {
String uniqueKey = wrapper.getIdempotent().uniqueKeyPrefix() + wrapper.getLockKey();
RLock lock = redissonClient.getLock(uniqueKey);
if (!lock.tryLock()) {
throw new ClientException(wrapper.getIdempotent().message());
}
IdempotentContext.put(LOCK, lock);
}
@Override
public void postProcessing() {
RLock lock = null;
try {
lock = (RLock) IdempotentContext.getKey(LOCK);
} finally {
if (lock != null) {
lock.unlock();
}
}
}
@Override
public void exceptionProcessing() {
RLock lock = null;
try {
lock = (RLock) IdempotentContext.getKey(LOCK);
} finally {
if (lock != null) {
lock.unlock();
}
}
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
# MQ场景
- handler(Wrapper):从wrapper中获取唯一key,查询缓存是否存在,如果存在,查询状态为,如果消费中,则抛出重复消费异常;如果不存在向本地线程中写入wrapper;
- exceptionProcessing():从本地线程中获取wrapper,获取唯一key,删除锁标识;
- postProcessing():从本地线程中获取wrapper,获取唯一key,写入缓存(消费完状态);
@RequiredArgsConstructor
public final class IdempotentSpELByMQExecuteHandler extends AbstractIdempotentExecuteHandler implements IdempotentSpELService {
private final DistributedCache distributedCache;
private final static int TIMEOUT = 600;
private final static String WRAPPER = "wrapper:spEL:MQ";
@SneakyThrows
@Override
protected IdempotentParamWrapper buildWrapper(ProceedingJoinPoint joinPoint) {
Idempotent idempotent = IdempotentAspect.getIdempotent(joinPoint);
String key = (String) SpELUtil.parseKey(idempotent.key(), ((MethodSignature) joinPoint.getSignature()).getMethod(), joinPoint.getArgs());
// = messageId+hashcode
return IdempotentParamWrapper.builder().lockKey(key).joinPoint(joinPoint).build();
}
@Override
public void handler(IdempotentParamWrapper wrapper) {
String uniqueKey = wrapper.getIdempotent().uniqueKeyPrefix() + wrapper.getLockKey();
Boolean setIfAbsent = ((StringRedisTemplate) distributedCache.getInstance())
.opsForValue()
.setIfAbsent(uniqueKey, IdempotentMQConsumeStatusEnum.CONSUMING.getCode(), TIMEOUT, TimeUnit.SECONDS);
if (setIfAbsent != null && !setIfAbsent) {
String consumeStatus = distributedCache.get(uniqueKey, String.class);
boolean error = IdempotentMQConsumeStatusEnum.isError(consumeStatus);
LogUtil.getLog(wrapper.getJoinPoint()).warn("[{}] MQ repeated consumption, {}.", uniqueKey, error ? "Wait for the client to delay consumption" : "Status is completed");
throw new RepeatConsumptionException(error);
}
IdempotentContext.put(WRAPPER, wrapper);
}
@Override
public void exceptionProcessing() {
IdempotentParamWrapper wrapper = (IdempotentParamWrapper) IdempotentContext.getKey(WRAPPER);
if (wrapper != null) {
Idempotent idempotent = wrapper.getIdempotent();
String uniqueKey = idempotent.uniqueKeyPrefix() + wrapper.getLockKey();
try {
distributedCache.delete(uniqueKey);
} catch (Throwable ex) {
LogUtil.getLog(wrapper.getJoinPoint()).error("[{}] Failed to del MQ anti-heavy token.", uniqueKey);
}
}
}
@Override
public void postProcessing() {
IdempotentParamWrapper wrapper = (IdempotentParamWrapper) IdempotentContext.getKey(WRAPPER);
if (wrapper != null) {
Idempotent idempotent = wrapper.getIdempotent();
String uniqueKey = idempotent.uniqueKeyPrefix() + wrapper.getLockKey();
try {
distributedCache.put(uniqueKey, IdempotentMQConsumeStatusEnum.CONSUMED.getCode(), idempotent.keyTimeout(), TimeUnit.SECONDS);
} catch (Throwable ex) {
LogUtil.getLog(wrapper.getJoinPoint()).error("[{}] Failed to set MQ anti-heavy token.", uniqueKey);
}
}
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
# Token方式幂等
需要前后端交互完成;
前端先向后端获取token,下一次请求携带token;
获取token
@RestController @RequiredArgsConstructor public class IdempotentTokenController { private final IdempotentTokenService idempotentTokenService; /** * 请求申请Token */ @GetMapping("/token") public Result<String> createToken() { return Results.success(idempotentTokenService.createToken()); } }
1
2
3
4
5
6
7
8
9
10
11
12
13
14public interface IdempotentTokenService extends IdempotentExecuteHandler { /** * 创建幂等验证Token */ String createToken(); }
1
2
3
4
5
6
7幂等处理器
服务端收到请求后,首先判断该token在redis中存在,若存在,完成业务逻辑后删除token;如果不存在(删除失败,抛出异常),代表是重复请求;
@RequiredArgsConstructor public final class IdempotentTokenExecuteHandler extends AbstractIdempotentExecuteHandler implements IdempotentTokenService { private final DistributedCache distributedCache; private final IdempotentProperties idempotentProperties; private static final String TOKEN_KEY = "token"; private static final String TOKEN_PREFIX_KEY = "idempotent:token:"; private static final long TOKEN_EXPIRED_TIME = 6000; @Override protected IdempotentParamWrapper buildWrapper(ProceedingJoinPoint joinPoint) { return new IdempotentParamWrapper(); } @Override public String createToken() { String token = Optional.ofNullable(Strings.emptyToNull(idempotentProperties.getPrefix())).orElse(TOKEN_PREFIX_KEY) + UUID.randomUUID(); distributedCache.put(token, "", Optional.ofNullable(idempotentProperties.getTimeout()).orElse(TOKEN_EXPIRED_TIME)); return token; } @Override public void handler(IdempotentParamWrapper wrapper) { HttpServletRequest request = ((ServletRequestAttributes) (RequestContextHolder.currentRequestAttributes())).getRequest(); String token = request.getHeader(TOKEN_KEY); if (StrUtil.isBlank(token)) { token = request.getParameter(TOKEN_KEY); if (StrUtil.isBlank(token)) { throw new ClientException(BaseErrorCode.IDEMPOTENT_TOKEN_NULL_ERROR); } } Boolean tokenDelFlag = distributedCache.delete(token); if (!tokenDelFlag) { String errMsg = StrUtil.isNotBlank(wrapper.getIdempotent().message()) ? wrapper.getIdempotent().message() : BaseErrorCode.IDEMPOTENT_TOKEN_DELETE_ERROR.message(); throw new ClientException(errMsg, BaseErrorCode.IDEMPOTENT_TOKEN_DELETE_ERROR); } } }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
先删除token,再执行业务!
如果业务代码执行超时,没有向客户端返回结果,客户端会进行重试;但是token已经删除,会被认为是重复请求,不再进行业务处理;
解决:让客户端重新获取令牌,重新发起一次访问;
先执行业务,再删除token!
高并发下,第一次访问token存在,完成业务,但是还没有删除token,客户端又携带token发起请求,因为token还存在,第二次请求也会验证通过;
解决:串行执行,对业务代码和删除token加锁;(不推荐)
# 参数方式幂等
@RequiredArgsConstructor
public final class IdempotentParamExecuteHandler extends AbstractIdempotentExecuteHandler implements IdempotentParamService {
private final RedissonClient redissonClient;
private final static String LOCK = "lock:param:restAPI";
@Override
protected IdempotentParamWrapper buildWrapper(ProceedingJoinPoint joinPoint) {
String lockKey = String.format("idempotent:path:%s:currentUserId:%s:md5:%s", getServletPath(), getCurrentUserId(), calcArgsMD5(joinPoint));
return IdempotentParamWrapper.builder().lockKey(lockKey).joinPoint(joinPoint).build();
}
/**
* @return 获取当前线程上下文 ServletPath
*/
private String getServletPath() {
ServletRequestAttributes sra = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
return sra.getRequest().getServletPath();
}
/**
* @return 当前操作用户 ID
*/
private String getCurrentUserId() {
return null;
}
/**
* @return joinPoint md5
*/
private String calcArgsMD5(ProceedingJoinPoint joinPoint) {
String md5 = DigestUtil.md5Hex(JSON.toJSONBytes(joinPoint.getArgs()));
return md5;
}
@Override
public void handler(IdempotentParamWrapper wrapper) {
String lockKey = wrapper.getLockKey();
RLock lock = redissonClient.getLock(lockKey);
if (!lock.tryLock()) {
throw new ClientException(wrapper.getIdempotent().message());
}
IdempotentContext.put(LOCK, lock);
}
@Override
public void postProcessing() {
RLock lock = null;
try {
lock = (RLock) IdempotentContext.getKey(LOCK);
} finally {
if (lock != null) {
lock.unlock();
}
}
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# 切面类
先获取注解,从处理器工厂中获取对应的处理器;
joinPoint.proceed()方法前,执行handler方法,方法后,执行postProcessiong方法,捕捉异常,异常中执行exceptionProcessing方法;
@Aspect
public final class IdempotentAspect {
/**
* 增强方法标记 {@link Idempotent} 注解逻辑
*/
@Around("@annotation(...framework.starter.idempotent.annotation.Idempotent)")
public Object idempotentHandler(ProceedingJoinPoint joinPoint) throws Throwable {
Idempotent idempotent = getIdempotent(joinPoint);
IdempotentExecuteHandler instance = IdempotentExecuteHandlerFactory.getInstance(idempotent.scene(), idempotent.type());
Object resultObj;
try {
instance.execute(joinPoint, idempotent);
resultObj = joinPoint.proceed();
instance.postProcessing(); // 在handler实现类定义,放入缓存中,key为uniqueKey,val为状态(消费完 消费中)
} catch (RepeatConsumptionException ex) {
/**
* 触发幂等逻辑时可能有两种情况:
* * 1. 消息还在处理,但是不确定是否执行成功,那么需要返回错误,方便 RocketMQ 再次通过重试队列投递
* * 2. 消息处理成功了,该消息直接返回成功即可
*/
if (!ex.getError()) {
return null;
}
throw ex;
} catch (Throwable ex) {
// 客户端消费存在异常,需要删除幂等标识方便下次 RocketMQ 再次通过重试队列投递
instance.exceptionProcessing();
throw ex;
} finally {
IdempotentContext.clean();
}
return resultObj;
}
public static Idempotent getIdempotent(ProceedingJoinPoint joinPoint) throws NoSuchMethodException {
MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature();
Method targetMethod = joinPoint.getTarget().getClass().getDeclaredMethod(methodSignature.getName(), methodSignature.getMethod().getParameterTypes());
return targetMethod.getAnnotation(Idempotent.class);
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# 实际场景使用
@Slf4j
@Component
@RequiredArgsConstructor
@RocketMQMessageListener(
topic = OrderRocketMQConstant.PAY_GLOBAL_TOPIC_KEY,
selectorExpression = OrderRocketMQConstant.PAY_RESULT_CALLBACK_ORDER_TAG_KEY,
consumerGroup = OrderRocketMQConstant.PAY_RESULT_CALLBACK_ORDER_CG_KEY
)
public class PayResultCallbackOrderConsumer implements RocketMQListener<MessageWrapper<PayResultCallbackOrderEvent>> {
private final OrderService orderService;
@Idempotent(
uniqueKeyPrefix = "order:pay_result_callback:",
key = "#message.getKeys()+'_'+#message.hashCode()",
type = IdempotentTypeEnum.SPEL,
scene = IdempotentSceneEnum.MQ,
keyTimeout = 7200L
)
@Transactional(rollbackFor = Exception.class)
@Override
public void onMessage(MessageWrapper<PayResultCallbackOrderEvent> message) {
PayResultCallbackOrderEvent payResultCallbackOrderEvent = message.getMessage();
OrderStatusReversalDTO orderStatusReversalDTO = OrderStatusReversalDTO.builder()
.orderSn(payResultCallbackOrderEvent.getOrderSn())
.orderStatus(OrderStatusEnum.ALREADY_PAID.getStatus())
.build();
orderService.statusReversal(orderStatusReversalDTO);
orderService.payCallbackOrder(payResultCallbackOrderEvent);
}
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31