Telegram-Bots/gitlab-notification
4
1
Fork 0
Code Issues 21 Pull Requests 1 Packages Projects 2 Releases 1 Activity

Оптимизация очистки старых mr

Browse Source
This commit is contained in:
Struchkov Mark 2022-12-20 21:01:49 +03:00
parent c47c1eca0f
commit a5e706a524
Signed by: upagge
GPG Key ID: D3018BE7BA428CA6
9 changed files with 29 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bot-context/src/main/java/dev/struchkov/bot/gitlab/context/repository/MergeRequestRepository.java
View File

@ -3,10 +3,7 @@ package dev.struchkov.bot.gitlab.context.repository;
import dev.struchkov.bot.gitlab.context.domain.IdAndStatusPr;
import dev.struchkov.bot.gitlab.context.domain.MergeRequestState;
import dev.struchkov.bot.gitlab.context.domain.entity.MergeRequest;
import dev.struchkov.haiti.filter.Filter;
import lombok.NonNull;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import java.util.List;
import java.util.Optional;
@ -26,8 +23,8 @@ public interface MergeRequestRepository {
void deleteByIds(Set<Long> mergeRequestIds);
Page<MergeRequest> filter(Filter filter, Pageable pageable);
List<MergeRequest> findAllByReviewerId(Long personId);
void deleteByStates(Set<MergeRequestState> states);
}

18
bot-context/src/main/java/dev/struchkov/bot/gitlab/context/service/CleanService.java
View File

@ -1,18 +0,0 @@
package dev.struchkov.bot.gitlab.context.service;
/**
* <p>Контракт очистки хранилища.</p>
* <p>Так как все что мы получаем от гитлаба сохраняется в БД, иногда нужно удалять устаревшие данные. Например, MR, которые уже были давно вмержены.</p>
*
* @author upagge 08.02.2021
*/
public interface CleanService {
/**
* <p>Удаляет старые MR.</p>
* <p>По умолчанию старыми считаются те, которые закрыты или вмержены</p>
*/
void cleanOldMergedRequests();
}

7
bot-context/src/main/java/dev/struchkov/bot/gitlab/context/service/MergeRequestsService.java
View File

@ -4,10 +4,7 @@ import dev.struchkov.bot.gitlab.context.domain.ExistContainer;
import dev.struchkov.bot.gitlab.context.domain.IdAndStatusPr;
import dev.struchkov.bot.gitlab.context.domain.MergeRequestState;
import dev.struchkov.bot.gitlab.context.domain.entity.MergeRequest;
import dev.struchkov.bot.gitlab.context.domain.filter.MergeRequestFilter;
import lombok.NonNull;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import java.util.List;
import java.util.Set;
@ -30,8 +27,6 @@ public interface MergeRequestsService {
List<MergeRequest> getAll();
Page<MergeRequest> getAll(@NonNull MergeRequestFilter filter, Pageable pagination);
ExistContainer<MergeRequest, Long> existsById(@NonNull Set<Long> mergeRequestIds);
List<MergeRequest> createAll(List<MergeRequest> newMergeRequests);
@ -40,4 +35,6 @@ public interface MergeRequestsService {
List<MergeRequest> getAllByReviewerId(@NonNull Long personId);
void cleanOld();
}

54
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/CleanServiceImpl.java
View File

@ -1,54 +0,0 @@
package dev.struchkov.bot.gitlab.core.service.impl;
import dev.struchkov.bot.gitlab.context.domain.entity.MergeRequest;
import dev.struchkov.bot.gitlab.context.domain.filter.MergeRequestFilter;
import dev.struchkov.bot.gitlab.context.service.CleanService;
import dev.struchkov.bot.gitlab.context.service.MergeRequestsService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;
import java.util.Set;
import java.util.stream.Collectors;
import static dev.struchkov.bot.gitlab.context.domain.MergeRequestState.CLOSED;
import static dev.struchkov.bot.gitlab.context.domain.MergeRequestState.MERGED;
/**
* Реализация сервиса очистки данных.
*
* @author upagge 08.02.2021
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class CleanServiceImpl implements CleanService {
private static final int COUNT = 1000;
private static final MergeRequestFilter MR_CLEAN_FILTER = MergeRequestFilter.builder()
.states(Set.of(MERGED, CLOSED))
.build();
private final MergeRequestsService mergeRequestsService;
@Override
public void cleanOldMergedRequests() {
log.debug("Старт очистки старых MR");
int page = 0;
Page<MergeRequest> mergeRequestSheet = mergeRequestsService.getAll(MR_CLEAN_FILTER, PageRequest.of(page, COUNT));
while (mergeRequestSheet.hasContent()) {
final Set<Long> ids = mergeRequestSheet.getContent().stream()
.map(MergeRequest::getId)
.collect(Collectors.toUnmodifiableSet());
mergeRequestsService.deleteAllById(ids);
mergeRequestSheet = mergeRequestsService.getAll(MR_CLEAN_FILTER, PageRequest.of(++page, COUNT));
}
log.debug("Конец очистки старых MR");
}
}

26
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/MergeRequestsServiceImpl.java
View File

@ -10,7 +10,6 @@ import dev.struchkov.bot.gitlab.context.domain.entity.Discussion;
import dev.struchkov.bot.gitlab.context.domain.entity.MergeRequest;
import dev.struchkov.bot.gitlab.context.domain.entity.Person;
import dev.struchkov.bot.gitlab.context.domain.entity.Project;
import dev.struchkov.bot.gitlab.context.domain.filter.MergeRequestFilter;
import dev.struchkov.bot.gitlab.context.domain.notify.mergerequest.ConflictPrNotify;
import dev.struchkov.bot.gitlab.context.domain.notify.mergerequest.NewPrNotify;
import dev.struchkov.bot.gitlab.context.domain.notify.mergerequest.StatusPrNotify;
@ -20,11 +19,9 @@ import dev.struchkov.bot.gitlab.context.service.DiscussionService;
import dev.struchkov.bot.gitlab.context.service.MergeRequestsService;
import dev.struchkov.bot.gitlab.context.service.NotifyService;
import dev.struchkov.bot.gitlab.context.service.ProjectService;
import dev.struchkov.bot.gitlab.core.service.impl.filter.MergeRequestFilterService;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@ -32,18 +29,23 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static dev.struchkov.bot.gitlab.context.domain.MergeRequestState.CLOSED;
import static dev.struchkov.bot.gitlab.context.domain.MergeRequestState.MERGED;
import static dev.struchkov.haiti.context.exception.NotFoundException.notFoundException;
import static dev.struchkov.haiti.utils.Checker.checkNotEmpty;
import static dev.struchkov.haiti.utils.Checker.checkNotNull;
import static java.lang.Boolean.TRUE;
@Slf4j
@Service
@RequiredArgsConstructor
public class MergeRequestsServiceImpl implements MergeRequestsService {
private final NotifyService notifyService;
public static final Set<MergeRequestState> DELETE_STATES = Set.of(MERGED, CLOSED);
private final MergeRequestRepository repository;
private final MergeRequestFilterService filterService;
private final NotifyService notifyService;
private final ProjectService projectService;
private final DiscussionService discussionService;
@ -214,11 +216,6 @@ public class MergeRequestsServiceImpl implements MergeRequestsService {
return repository.findAll();
}
@Override
public Page<MergeRequest> getAll(@NonNull MergeRequestFilter filter, Pageable pagination) {
return filterService.getAll(filter, pagination);
}
@Override
public ExistContainer<MergeRequest, Long> existsById(@NonNull Set<Long> mergeRequestIds) {
final List<MergeRequest> existsEntity = repository.findAllById(mergeRequestIds);
@ -251,6 +248,13 @@ public class MergeRequestsServiceImpl implements MergeRequestsService {
return repository.findAllByReviewerId(personId);
}
@Override
public void cleanOld() {
log.debug("Старт очистки старых MR");
repository.deleteByStates(DELETE_STATES);
log.debug("Конец очистки старых MR");
}
private void notifyAboutUpdate(MergeRequest oldMergeRequest, MergeRequest mergeRequest, Project project) {
final Long botUserGitlabId = personInformation.getId();

43
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/filter/MergeRequestFilterService.java
View File

@ -1,43 +0,0 @@
package dev.struchkov.bot.gitlab.core.service.impl.filter;
import dev.struchkov.bot.gitlab.context.domain.entity.MergeRequest;
import dev.struchkov.bot.gitlab.context.domain.entity.MergeRequestFields;
import dev.struchkov.bot.gitlab.context.domain.filter.MergeRequestFilter;
import dev.struchkov.bot.gitlab.context.repository.MergeRequestRepository;
import dev.struchkov.haiti.filter.Filter;
import dev.struchkov.haiti.filter.FilterQuery;
import dev.struchkov.haiti.filter.criteria.CriteriaFilter;
import dev.struchkov.haiti.filter.criteria.CriteriaQuery;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class MergeRequestFilterService {
private final MergeRequestRepository repository;
public Page<MergeRequest> getAll(MergeRequestFilter filter, Pageable pagination) {
return repository.filter(createFilter(filter), pagination);
}
private Filter createFilter(@NonNull MergeRequestFilter filter) {
return CriteriaFilter.<MergeRequest>create()
.and(convertFilter(filter))
.or(convertFilterOr(filter));
}
private FilterQuery convertFilterOr(MergeRequestFilter filter) {
return CriteriaQuery.<MergeRequest>create()
.matchPhrase(MergeRequestFields.state, filter.getStates());
}
private FilterQuery convertFilter(@NonNull MergeRequestFilter filter) {
return CriteriaQuery.<MergeRequest>create()
.matchPhrase(MergeRequestFields.assignee, filter.getAssignee());
}
}

14
bot-data/src/main/java/dev/struchkov/bot/gitlab/data/impl/MergeRequestRepositoryImpl.java
View File

@ -5,12 +5,8 @@ import dev.struchkov.bot.gitlab.context.domain.MergeRequestState;
import dev.struchkov.bot.gitlab.context.domain.entity.MergeRequest;
import dev.struchkov.bot.gitlab.context.repository.MergeRequestRepository;
import dev.struchkov.bot.gitlab.data.jpa.MergeRequestJpaRepository;
import dev.struchkov.haiti.filter.Filter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.stereotype.Repository;
import java.util.List;
@ -53,14 +49,14 @@ public class MergeRequestRepositoryImpl implements MergeRequestRepository {
jpaRepository.deleteAllByIdIn(mergeRequestIds);
}
@Override
public Page<MergeRequest> filter(Filter filter, Pageable pageable) {
return jpaRepository.findAll(filter.<Specification<MergeRequest>>build(), pageable);
}
@Override
public List<MergeRequest> findAllByReviewerId(Long personId) {
return jpaRepository.findAllByReviewersIn(personId);
}
@Override
public void deleteByStates(Set<MergeRequestState> states) {
jpaRepository.deleteAllByStateIn(states);
}
}

2
bot-data/src/main/java/dev/struchkov/bot/gitlab/data/jpa/MergeRequestJpaRepository.java
View File

@ -34,4 +34,6 @@ public interface MergeRequestJpaRepository extends JpaRepositoryImplementation<M
@Query("SELECT mr FROM MergeRequest mr LEFT JOIN mr.reviewers r WHERE r.id = :reviewerId")
List<MergeRequest> findAllByReviewersIn(@Param("reviewerId") Long reviewerId);
void deleteAllByStateIn(Set<MergeRequestState> states);
}

6
gitlab-app/src/main/java/dev/struchkov/bot/gitlab/scheduler/SchedulerService.java
View File

@ -1,6 +1,6 @@
package dev.struchkov.bot.gitlab.scheduler;
import dev.struchkov.bot.gitlab.context.service.CleanService;
import dev.struchkov.bot.gitlab.context.service.MergeRequestsService;
import dev.struchkov.bot.gitlab.context.service.PipelineService;
import dev.struchkov.bot.gitlab.core.service.parser.DiscussionParser;
import dev.struchkov.bot.gitlab.core.service.parser.MergeRequestParser;
@ -20,10 +20,10 @@ public class SchedulerService {
private final PipelineParser pipelineParser;
private final MergeRequestParser mergeRequestParser;
private final CleanService cleanService;
private final DiscussionParser discussionParser;
private final PipelineService pipelineService;
private final MergeRequestsService mergeRequestsService;
@Scheduled(cron = "0 */1 * * * *")
public void newMergeRequest() {
@ -34,8 +34,8 @@ public class SchedulerService {
pipelineParser.scanNewPipeline();
discussionParser.scanOldDiscussions();
discussionParser.scanNewDiscussion();
mergeRequestsService.cleanOld();
pipelineService.cleanOld();
cleanService.cleanOldMergedRequests();
log.debug("Конец процесса обновления данных");
}