Telegram-Bots/gitlab-notification
4
1
Fork 0
Code Issues 21 Pull Requests 1 Packages Projects 2 Releases 1 Activity

Оптимизация очистки старых пайплайнов

Browse Source
This commit is contained in:
Struchkov Mark 2022-12-20 20:49:42 +03:00
parent 0980b05925
commit c47c1eca0f
Signed by: upagge
GPG Key ID: D3018BE7BA428CA6
9 changed files with 23 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bot-context/src/main/java/dev/struchkov/bot/gitlab/context/repository/PipelineRepository.java
View File

@ -2,10 +2,8 @@ package dev.struchkov.bot.gitlab.context.repository;
import dev.struchkov.bot.gitlab.context.domain.PipelineStatus;
import dev.struchkov.bot.gitlab.context.domain.entity.Pipeline;
import dev.struchkov.haiti.filter.Filter;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@ -23,8 +21,6 @@ public interface PipelineRepository {
List<Pipeline> findAllById(Set<Long> pipelineIds);
void deleteAllByIds(Set<Long> pipelineIds);
Page<Pipeline> filter(Filter filter, Pageable pagination);
void deleteByCreatedBefore(LocalDateTime date);
}

5
bot-context/src/main/java/dev/struchkov/bot/gitlab/context/service/CleanService.java
View File

@ -14,10 +14,5 @@ public interface CleanService {
*/
void cleanOldMergedRequests();
/**
* <p>Удаляет старые пайплайны</p>
* <p>По умолчанию удаляет все пайплайны старше суток</p>
*/
void cleanOldPipelines();
}

8
bot-context/src/main/java/dev/struchkov/bot/gitlab/context/service/PipelineService.java
View File

@ -3,10 +3,7 @@ package dev.struchkov.bot.gitlab.context.service;
import dev.struchkov.bot.gitlab.context.domain.ExistContainer;
import dev.struchkov.bot.gitlab.context.domain.PipelineStatus;
import dev.struchkov.bot.gitlab.context.domain.entity.Pipeline;
import dev.struchkov.bot.gitlab.context.domain.filter.PipelineFilter;
import lombok.NonNull;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import java.util.List;
import java.util.Set;
@ -28,9 +25,8 @@ public interface PipelineService {
List<Pipeline> getAllByStatuses(@NonNull Set<PipelineStatus> statuses);
Page<Pipeline> getAll(@NonNull PipelineFilter filter, @NonNull Pageable pagination);
ExistContainer<Pipeline, Long> existsById(@NonNull Set<Long> pipelineIds);
void deleteAllById(Set<Long> pipelineIds);
void cleanOld();
}

30
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/CleanServiceImpl.java
View File

@ -1,19 +1,15 @@
package dev.struchkov.bot.gitlab.core.service.impl;
import dev.struchkov.bot.gitlab.context.domain.entity.MergeRequest;
import dev.struchkov.bot.gitlab.context.domain.entity.Pipeline;
import dev.struchkov.bot.gitlab.context.domain.filter.MergeRequestFilter;
import dev.struchkov.bot.gitlab.context.domain.filter.PipelineFilter;
import dev.struchkov.bot.gitlab.context.service.CleanService;
import dev.struchkov.bot.gitlab.context.service.MergeRequestsService;
import dev.struchkov.bot.gitlab.context.service.PipelineService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.Set;
import java.util.stream.Collectors;
@ -36,7 +32,6 @@ public class CleanServiceImpl implements CleanService {
.build();
private final MergeRequestsService mergeRequestsService;
private final PipelineService pipelineService;
@Override
public void cleanOldMergedRequests() {
@ -56,29 +51,4 @@ public class CleanServiceImpl implements CleanService {
log.debug("Конец очистки старых MR");
}
@Override
public void cleanOldPipelines() {
log.debug("Старт очистки старых пайплайнов");
int page = 0;
final PipelineFilter filter = cleanPipelineFilter();
Page<Pipeline> sheet = pipelineService.getAll(filter, PageRequest.of(page, COUNT));
while (sheet.hasContent()) {
final Set<Long> ids = sheet.getContent().stream()
.map(Pipeline::getId)
.collect(Collectors.toUnmodifiableSet());
pipelineService.deleteAllById(ids);
sheet = pipelineService.getAll(filter, PageRequest.of(page, COUNT));
}
log.debug("Конец очистки старых пайплайнов");
}
private PipelineFilter cleanPipelineFilter() {
final PipelineFilter pipelineFilter = new PipelineFilter();
pipelineFilter.setLessThanCreatedDate(LocalDateTime.now().minusDays(1L));
return pipelineFilter;
}
}

18
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/PipelineServiceImpl.java
View File

@ -5,19 +5,17 @@ import dev.struchkov.bot.gitlab.context.domain.PersonInformation;
import dev.struchkov.bot.gitlab.context.domain.PipelineStatus;
import dev.struchkov.bot.gitlab.context.domain.entity.Person;
import dev.struchkov.bot.gitlab.context.domain.entity.Pipeline;
import dev.struchkov.bot.gitlab.context.domain.filter.PipelineFilter;
import dev.struchkov.bot.gitlab.context.domain.notify.pipeline.PipelineNotify;
import dev.struchkov.bot.gitlab.context.repository.PipelineRepository;
import dev.struchkov.bot.gitlab.context.service.NotifyService;
import dev.struchkov.bot.gitlab.context.service.PipelineService;
import dev.struchkov.bot.gitlab.core.service.impl.filter.PipelineFilterService;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@ -34,6 +32,7 @@ import static dev.struchkov.haiti.utils.Checker.checkNotNull;
*
* @author upagge 17.01.2021
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class PipelineServiceImpl implements PipelineService {
@ -43,7 +42,6 @@ public class PipelineServiceImpl implements PipelineService {
private final NotifyService notifyService;
private final PipelineRepository repository;
private final PipelineFilterService pipelineFilterService;
private final PersonInformation personInformation;
@ -111,10 +109,6 @@ public class PipelineServiceImpl implements PipelineService {
return repository.findAllByStatuses(statuses);
}
@Override
public Page<Pipeline> getAll(@NonNull PipelineFilter filter, @NonNull Pageable pagination) {
return pipelineFilterService.getAll(filter, pagination);
}
@Override
public ExistContainer<Pipeline, Long> existsById(@NonNull Set<Long> pipelineIds) {
@ -131,8 +125,10 @@ public class PipelineServiceImpl implements PipelineService {
}
@Override
public void deleteAllById(Set<Long> pipelineIds) {
repository.deleteAllByIds(pipelineIds);
public void cleanOld() {
log.debug("Старт очистки старых пайплайнов");
repository.deleteByCreatedBefore(LocalDateTime.now().minusDays(1L));
log.debug("Конец очистки старых пайплайнов");
}
}

43
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/filter/PipelineFilterService.java
View File

@ -1,43 +0,0 @@
package dev.struchkov.bot.gitlab.core.service.impl.filter;
import dev.struchkov.bot.gitlab.context.domain.entity.Pipeline;
import dev.struchkov.bot.gitlab.context.domain.entity.PipelineFields;
import dev.struchkov.bot.gitlab.context.domain.filter.PipelineFilter;
import dev.struchkov.bot.gitlab.context.repository.PipelineRepository;
import dev.struchkov.haiti.filter.Filter;
import dev.struchkov.haiti.filter.FilterQuery;
import dev.struchkov.haiti.filter.criteria.CriteriaFilter;
import dev.struchkov.haiti.filter.criteria.CriteriaQuery;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
/**
* Сервис фильтрации пайплайнов.
*
* @author upagge 08.02.2021
*/
@Service
@RequiredArgsConstructor
public class PipelineFilterService {
private final PipelineRepository pipelineRepository;
public Page<Pipeline> getAll(PipelineFilter filter, Pageable pagination) {
return pipelineRepository.filter(createFilter(filter), pagination);
}
private Filter createFilter(@NonNull PipelineFilter pipelineFilter) {
return CriteriaFilter.<Pipeline>create()
.and(convertAnd(pipelineFilter));
}
private FilterQuery convertAnd(PipelineFilter pipelineFilter) {
return CriteriaQuery.<Pipeline>create()
.lessThan(PipelineFields.created, pipelineFilter.getLessThanCreatedDate());
}
}

16
bot-data/src/main/java/dev/struchkov/bot/gitlab/data/impl/PipelineRepositoryImpl.java
View File

@ -4,13 +4,11 @@ import dev.struchkov.bot.gitlab.context.domain.PipelineStatus;
import dev.struchkov.bot.gitlab.context.domain.entity.Pipeline;
import dev.struchkov.bot.gitlab.context.repository.PipelineRepository;
import dev.struchkov.bot.gitlab.data.jpa.PipelineJpaRepository;
import dev.struchkov.haiti.filter.Filter;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@ -45,13 +43,9 @@ public class PipelineRepositoryImpl implements PipelineRepository {
}
@Override
public void deleteAllByIds(Set<Long> pipelineIds) {
jpaRepository.deleteAllById(pipelineIds);
}
@Override
public Page<Pipeline> filter(Filter filter, Pageable pagination) {
return jpaRepository.findAll(filter.<Specification<Pipeline>>build(), pagination);
@Transactional
public void deleteByCreatedBefore(LocalDateTime date) {
jpaRepository.deleteAllByCreatedBefore(date);
}
}

3
bot-data/src/main/java/dev/struchkov/bot/gitlab/data/jpa/PipelineJpaRepository.java
View File

@ -4,6 +4,7 @@ import dev.struchkov.bot.gitlab.context.domain.PipelineStatus;
import dev.struchkov.bot.gitlab.context.domain.entity.Pipeline;
import org.springframework.data.jpa.repository.support.JpaRepositoryImplementation;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
@ -14,4 +15,6 @@ public interface PipelineJpaRepository extends JpaRepositoryImplementation<Pipel
List<Pipeline> findAllByStatusIn(Set<PipelineStatus> statuses);
void deleteAllByCreatedBefore(LocalDateTime date);
}

5
gitlab-app/src/main/java/dev/struchkov/bot/gitlab/scheduler/SchedulerService.java
View File

@ -1,6 +1,7 @@
package dev.struchkov.bot.gitlab.scheduler;
import dev.struchkov.bot.gitlab.context.service.CleanService;
import dev.struchkov.bot.gitlab.context.service.PipelineService;
import dev.struchkov.bot.gitlab.core.service.parser.DiscussionParser;
import dev.struchkov.bot.gitlab.core.service.parser.MergeRequestParser;
import dev.struchkov.bot.gitlab.core.service.parser.PipelineParser;
@ -22,6 +23,8 @@ public class SchedulerService {
private final CleanService cleanService;
private final DiscussionParser discussionParser;
private final PipelineService pipelineService;
@Scheduled(cron = "0 */1 * * * *")
public void newMergeRequest() {
log.debug("Запуск процесса обновления данных");
@ -31,7 +34,7 @@ public class SchedulerService {
pipelineParser.scanNewPipeline();
discussionParser.scanOldDiscussions();
discussionParser.scanNewDiscussion();
cleanService.cleanOldPipelines();
pipelineService.cleanOld();
cleanService.cleanOldMergedRequests();
log.debug("Конец процесса обновления данных");
}