theonedev/onedev
1
0
Fork 0
mirror of https://github.com/theonedev/onedev.git synced 2025-12-10 00:07:37 -06:00
Code Issues Packages Projects Releases 429 Wiki Activity

feat: Improve startup speed when there are many projects (OD-2509)

Browse Source
This commit is contained in:
Robin Shen 2025-08-01 12:56:16 +08:00
parent af58b47205
commit b46dfb8637
14 changed files with 295 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
View File

@ -8159,4 +8159,26 @@ public class DataMigrator {
}
}
private void migrate207(File dataDir, Stack<Integer> versions) {
for (File file : dataDir.listFiles()) {
if (file.getName().startsWith("Projects.xml")) {
VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);
for (Element element : dom.getRootElement().elements()) {
element.element("lastEventDate").setName("lastActivityDate");
}
dom.writeToFile(file, false);
} else if (file.getName().startsWith("ProjectLastEventDates.xml")) {
VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);
for (Element element : dom.getRootElement().elements()) {
element.setName("io.onedev.server.model.ProjectLastActivityDate");
var commitElement = element.element("commit");
if (commitElement != null)
commitElement.detach();
element.element("activity").setName("value");
}
dom.writeToFile(new File(dataDir, file.getName().replace("ProjectLastEventDates", "ProjectLastActivityDates")), false);
}
}
}
}

6
server-core/src/main/java/io/onedev/server/entitymanager/ProjectLastEventDateManager.java
View File

@ -1,10 +1,10 @@
package io.onedev.server.entitymanager;
import io.onedev.server.model.ProjectLastEventDate;
import io.onedev.server.model.ProjectLastActivityDate;
import io.onedev.server.persistence.dao.EntityManager;
public interface ProjectLastEventDateManager extends EntityManager<ProjectLastEventDate> {
public interface ProjectLastEventDateManager extends EntityManager<ProjectLastActivityDate> {
void create(ProjectLastEventDate lastEventDate);
void create(ProjectLastActivityDate lastEventDate);
}

11
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectLastEventDateManager.java
View File

@ -7,7 +7,7 @@ import io.onedev.server.event.project.ProjectCreated;
import io.onedev.server.event.project.ProjectEvent;
import io.onedev.server.event.project.RefUpdated;
import io.onedev.server.model.Project;
import io.onedev.server.model.ProjectLastEventDate;
import io.onedev.server.model.ProjectLastActivityDate;
import io.onedev.server.persistence.annotation.Transactional;
import io.onedev.server.persistence.dao.BaseEntityManager;
import io.onedev.server.persistence.dao.Dao;
@ -17,7 +17,7 @@ import javax.inject.Singleton;
import java.util.Date;
@Singleton
public class DefaultProjectLastEventDateManager extends BaseEntityManager<ProjectLastEventDate> implements ProjectLastEventDateManager {
public class DefaultProjectLastEventDateManager extends BaseEntityManager<ProjectLastActivityDate> implements ProjectLastEventDateManager {
@Inject
public DefaultProjectLastEventDateManager(Dao dao) {
@ -29,18 +29,17 @@ public class DefaultProjectLastEventDateManager extends BaseEntityManager<Projec
public void on(ProjectEvent event) {
Project project = event.getProject();
if (event instanceof RefUpdated) {
project.getLastEventDate().setActivity(new Date());
project.getLastEventDate().setCommit(new Date());
project.getLastActivityDate().setValue(new Date());
} else if (!(event instanceof ProjectCreated)
&& event.getUser() != null
&& !event.getUser().isSystem()) {
project.getLastEventDate().setActivity(new Date());
project.getLastActivityDate().setValue(new Date());
}
}
@Transactional
@Override
public void create(ProjectLastEventDate lastEventDate) {
public void create(ProjectLastActivityDate lastEventDate) {
Preconditions.checkState(lastEventDate.isNew());
dao.persist(lastEventDate);
}

33
server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
View File

@ -5,7 +5,6 @@ import static io.onedev.commons.utils.LockUtils.read;
import static io.onedev.k8shelper.KubernetesHelper.BEARER;
import static io.onedev.server.git.CommandUtils.callWithClusterCredential;
import static io.onedev.server.git.GitUtils.getDefaultBranch;
import static io.onedev.server.git.GitUtils.getLastCommit;
import static io.onedev.server.git.GitUtils.getReachableCommits;
import static io.onedev.server.git.GitUtils.isValid;
import static io.onedev.server.git.GitUtils.setDefaultBranch;
@ -87,7 +86,6 @@ import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.revwalk.RevCommit;
import org.hibernate.Session;
import org.hibernate.criterion.Restrictions;
import org.hibernate.query.Query;
@ -160,7 +158,7 @@ import io.onedev.server.model.LinkSpec;
import io.onedev.server.model.Pack;
import io.onedev.server.model.PackBlob;
import io.onedev.server.model.Project;
import io.onedev.server.model.ProjectLastEventDate;
import io.onedev.server.model.ProjectLastActivityDate;
import io.onedev.server.model.PullRequest;
import io.onedev.server.model.User;
import io.onedev.server.model.UserAuthorization;
@ -383,8 +381,8 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
}
project.setPath(project.calcPath());
ProjectLastEventDate lastEventDate = new ProjectLastEventDate();
project.setLastEventDate(lastEventDate);
ProjectLastActivityDate lastEventDate = new ProjectLastActivityDate();
project.setLastActivityDate(lastEventDate);
lastEventDateManager.create(lastEventDate);
dao.persist(project);
@ -519,7 +517,7 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
packBlobManager.delete(packBlob);
dao.remove(project);
lastEventDateManager.delete(project.getLastEventDate());
lastEventDateManager.delete(project.getLastActivityDate());
synchronized (repositoryCache) {
Repository repository = repositoryCache.remove(project.getId());
@ -622,7 +620,6 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
@Transactional
@Override
public void fork(Project from, Project to) {
to.getLastEventDate().setCommit(from.getLastEventDate().getCommit());
Long fromId = from.getId();
String fromPath = from.getPath();
Long toId = to.getId();
@ -816,8 +813,8 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
cache.put(project.getId(), project.getFacade());
}
Map<Long, ProjectLastEventDate> lastEventDates = new HashMap<>();
for (ProjectLastEventDate lastEventDate : lastEventDateManager.query())
Map<Long, ProjectLastActivityDate> lastEventDates = new HashMap<>();
for (ProjectLastActivityDate lastEventDate : lastEventDateManager.query())
lastEventDates.put(lastEventDate.getId(), lastEventDate);
logger.info("Checking projects...");
@ -842,16 +839,6 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
checkGitDir(projectId);
HookUtils.checkHooks(getGitDir(projectId));
checkGitConfig(projectId, project.getGitPackConfig());
if (project.isCodeManagement()) {
ProjectLastEventDate lastEventDate = lastEventDates.get(project.getLastEventDateId());
RevCommit lastCommit = getLastCommit(getRepository(projectId));
if (lastCommit != null) {
var lastCommitDate = lastCommit.getCommitterIdent().getWhen();
if (lastEventDate.getCommit() == null || lastEventDate.getCommit().before(lastCommitDate))
lastEventDate.setCommit(lastCommitDate);
}
}
LinkedHashMap<String, ProjectReplica> newReplicasOfProject;
var replica = new ProjectReplica();
@ -877,7 +864,7 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
throw new RuntimeException(e);
}
}
updateActiveServer(projectId, newReplicasOfProject, false);
updateActiveServer(projectId, newReplicasOfProject, false);
}
}
}
@ -1062,16 +1049,16 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
for (var order: orders) {
if (order.getExpression() instanceof SingularAttributePath) {
var expr = (SingularAttributePath) order.getExpression();
if (expr.getAttribute().getName().equals(ProjectLastEventDate.PROP_ACTIVITY)
if (expr.getAttribute().getName().equals(ProjectLastActivityDate.PROP_VALUE)
&& expr.getPathSource() instanceof SingularAttributePath
&& ((SingularAttributePath) expr.getPathSource()).getAttribute().getName().equals(Project.PROP_LAST_EVENT_DATE)) {
&& ((SingularAttributePath) expr.getPathSource()).getAttribute().getName().equals(Project.PROP_LAST_ACTIVITY_DATE)) {
found = true;
break;
}
}
}
if (!found)
orders.add(builder.desc(ProjectQuery.getPath(root, Project.PROP_LAST_EVENT_DATE + "." + ProjectLastEventDate.PROP_ACTIVITY)));
orders.add(builder.desc(ProjectQuery.getPath(root, Project.PROP_LAST_ACTIVITY_DATE + "." + ProjectLastActivityDate.PROP_VALUE)));
query.orderBy(orders);

23
server-core/src/main/java/io/onedev/server/model/Project.java
View File

@ -145,7 +145,7 @@ import io.onedev.server.xodus.CommitInfoManager;
@Table(
indexes={
@Index(columnList="o_parent_id"), @Index(columnList="o_forkedFrom_id"),
@Index(columnList="o_lastEventDate_id"), @Index(columnList=PROP_NAME),
@Index(columnList="o_lastActivityDate_id"), @Index(columnList=PROP_NAME),
@Index(columnList=PROP_PATH)
},
uniqueConstraints={@UniqueConstraint(columnNames={"o_parent_id", PROP_NAME})}
@ -187,10 +187,8 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
public static final String PROP_FORKED_FROM = "forkedFrom";
public static final String NAME_LAST_ACTIVITY_DATE = "Last Activity Date";
public static final String NAME_LAST_COMMIT_DATE = "Last Commit Date";
public static final String PROP_LAST_EVENT_DATE = "lastEventDate";
public static final String PROP_LAST_ACTIVITY_DATE = "lastActivityDate";
public static final String NAME_LABEL = "Label";
@ -219,7 +217,7 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
private static final String FAKED_GITHUB_REPO_OWNER = "faked-owner-of-onedev-project";
public static final List<String> QUERY_FIELDS = Lists.newArrayList(
NAME_NAME, NAME_KEY, NAME_PATH, NAME_LABEL, NAME_SERVICE_DESK_EMAIL_ADDRESS, NAME_ID, NAME_DESCRIPTION, NAME_LAST_ACTIVITY_DATE, NAME_LAST_COMMIT_DATE);
NAME_NAME, NAME_KEY, NAME_PATH, NAME_LABEL, NAME_SERVICE_DESK_EMAIL_ADDRESS, NAME_ID, NAME_DESCRIPTION, NAME_LAST_ACTIVITY_DATE);
public static final Map<String, SortField<Project>> SORT_FIELDS = new LinkedHashMap<>();
static {
@ -228,8 +226,7 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
SORT_FIELDS.put(NAME_NAME, new SortField<>(PROP_NAME));
SORT_FIELDS.put(NAME_KEY, new SortField<>(PROP_KEY));
SORT_FIELDS.put(NAME_SERVICE_DESK_EMAIL_ADDRESS, new SortField<>(PROP_SERVICE_DESK_EMAIL_ADDRESS));
SORT_FIELDS.put(NAME_LAST_ACTIVITY_DATE, new SortField<>(PROP_LAST_EVENT_DATE + "." + ProjectLastEventDate.PROP_ACTIVITY, DESCENDING));
SORT_FIELDS.put(NAME_LAST_COMMIT_DATE, new SortField<>(PROP_LAST_EVENT_DATE + "." + ProjectLastEventDate.PROP_COMMIT, DESCENDING));
SORT_FIELDS.put(NAME_LAST_ACTIVITY_DATE, new SortField<>(PROP_LAST_ACTIVITY_DATE + "." + ProjectLastActivityDate.PROP_VALUE, DESCENDING));
}
static ThreadLocal<Stack<Project>> stack = ThreadLocal.withInitial(() -> new Stack<>());
@ -257,7 +254,7 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(unique=true, nullable=false)
private ProjectLastEventDate lastEventDate;
private ProjectLastActivityDate lastActivityDate;
@Column(nullable=false)
private String name;
@ -546,12 +543,12 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
this.createDate = createDate;
}
public ProjectLastEventDate getLastEventDate() {
return lastEventDate;
public ProjectLastActivityDate getLastActivityDate() {
return lastActivityDate;
}
public void setLastEventDate(ProjectLastEventDate lastEventDate) {
this.lastEventDate = lastEventDate;
public void setLastActivityDate(ProjectLastActivityDate lastActivityDate) {
this.lastActivityDate = lastActivityDate;
}
public Collection<PullRequest> getIncomingRequests() {
@ -756,7 +753,7 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
public ProjectFacade getFacade() {
return new ProjectFacade(getId(), getName(), getKey(), getPath(), getServiceDeskEmailAddress(),
isCodeManagement(), isIssueManagement(), getGitPackConfig(), lastEventDate.getId(),
isCodeManagement(), isIssueManagement(), getGitPackConfig(), lastActivityDate.getId(),
idOf(getParent()), idOf(getForkedFrom()));
}

35
server-core/src/main/java/io/onedev/server/model/ProjectLastActivityDate.java Normal file
View File

@ -0,0 +1,35 @@
package io.onedev.server.model;
import static io.onedev.server.model.ProjectLastActivityDate.PROP_VALUE;
import java.util.Date;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Index;
import javax.persistence.Table;
/**
* Maintain high dynamic data in a separate table to avoid project second-level
* cache being invalidated frequently
*/
@Entity
@Table(indexes={@Index(columnList= PROP_VALUE)})
public class ProjectLastActivityDate extends AbstractEntity {
private static final long serialVersionUID = 1L;
public static final String PROP_VALUE = "value";
@Column(nullable=false)
private Date value = new Date();
public Date getValue() {
return value;
}
public void setValue(Date value) {
this.value = value;
}
}

49
server-core/src/main/java/io/onedev/server/model/ProjectLastEventDate.java
View File

@ -1,49 +0,0 @@
package io.onedev.server.model;
import javax.annotation.Nullable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Index;
import javax.persistence.Table;
import java.util.Date;
import static io.onedev.server.model.ProjectLastEventDate.PROP_ACTIVITY;
import static io.onedev.server.model.ProjectLastEventDate.PROP_COMMIT;
/**
* Maintain high dynamic data in a separate table to avoid project second-level
* cache being invalidated frequently
*/
@Entity
@Table(indexes={@Index(columnList= PROP_ACTIVITY), @Index(columnList= PROP_COMMIT)})
public class ProjectLastEventDate extends AbstractEntity {
private static final long serialVersionUID = 1L;
public static final String PROP_ACTIVITY = "activity";
public static final String PROP_COMMIT = "commit";
@Column(nullable=false)
private Date activity = new Date();
private Date commit;
public Date getActivity() {
return activity;
}
public void setActivity(Date activity) {
this.activity = activity;
}
@Nullable
public Date getCommit() {
return commit;
}
public void setCommit(@Nullable Date commit) {
this.commit = commit;
}
}

176
server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.java
View File

@ -1,55 +1,23 @@
package io.onedev.server.search.code;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import io.onedev.commons.jsymbol.Symbol;
import io.onedev.commons.jsymbol.SymbolExtractor;
import io.onedev.commons.jsymbol.SymbolExtractorRegistry;
import io.onedev.commons.loader.ManagedSerializedForm;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.StringUtils;
import io.onedev.server.cluster.ClusterTask;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.event.Listen;
import io.onedev.server.event.ListenerRegistry;
import io.onedev.server.event.project.CommitIndexed;
import io.onedev.server.event.project.RefUpdated;
import io.onedev.server.event.system.SystemStarted;
import io.onedev.server.git.GitUtils;
import io.onedev.server.model.Project;
import io.onedev.server.persistence.SessionManager;
import io.onedev.server.persistence.annotation.Sessional;
import io.onedev.server.util.ContentDetector;
import io.onedev.server.util.IndexResult;
import io.onedev.server.util.concurrent.BatchWorkManager;
import io.onedev.server.util.concurrent.BatchWorker;
import io.onedev.server.util.concurrent.Prioritized;
import io.onedev.commons.utils.match.Matcher;
import io.onedev.commons.utils.match.PathMatcher;
import io.onedev.server.util.patternset.PatternSet;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.lucene.document.*;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.wicket.request.cycle.RequestCycle;
import org.eclipse.jgit.lib.*;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.TreeFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static io.onedev.server.search.code.FieldConstants.BLOB_HASH;
import static io.onedev.server.search.code.FieldConstants.BLOB_INDEX_VERSION;
import static io.onedev.server.search.code.FieldConstants.BLOB_NAME;
import static io.onedev.server.search.code.FieldConstants.BLOB_PATH;
import static io.onedev.server.search.code.FieldConstants.BLOB_PRIMARY_SYMBOLS;
import static io.onedev.server.search.code.FieldConstants.BLOB_SECONDARY_SYMBOLS;
import static io.onedev.server.search.code.FieldConstants.BLOB_SYMBOL_LIST;
import static io.onedev.server.search.code.FieldConstants.BLOB_TEXT;
import static io.onedev.server.search.code.FieldConstants.COMMIT_HASH;
import static io.onedev.server.search.code.FieldConstants.COMMIT_INDEX_VERSION;
import static io.onedev.server.search.code.FieldConstants.LAST_COMMIT;
import static io.onedev.server.search.code.FieldConstants.LAST_COMMIT_HASH;
import static io.onedev.server.search.code.FieldConstants.LAST_COMMIT_INDEX_VERSION;
import static io.onedev.server.search.code.FieldConstants.META;
import static io.onedev.server.search.code.IndexConstants.MAX_INDEXABLE_BLOB_SIZE;
import static io.onedev.server.search.code.IndexConstants.MAX_INDEXABLE_LINE_LEN;
import static io.onedev.server.search.code.IndexConstants.NGRAM_SIZE;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.File;
import java.io.IOException;
import java.io.ObjectStreamException;
@ -58,8 +26,70 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import static io.onedev.server.search.code.FieldConstants.*;
import static io.onedev.server.search.code.IndexConstants.*;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.SerializationUtils;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.wicket.request.cycle.RequestCycle;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.TreeFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import io.onedev.commons.jsymbol.Symbol;
import io.onedev.commons.jsymbol.SymbolExtractor;
import io.onedev.commons.jsymbol.SymbolExtractorRegistry;
import io.onedev.commons.loader.ManagedSerializedForm;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.match.Matcher;
import io.onedev.commons.utils.match.PathMatcher;
import io.onedev.server.cluster.ClusterTask;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.event.ListenerRegistry;
import io.onedev.server.event.project.CommitIndexed;
import io.onedev.server.git.GitUtils;
import io.onedev.server.model.Project;
import io.onedev.server.persistence.SessionManager;
import io.onedev.server.util.ContentDetector;
import io.onedev.server.util.IndexResult;
import io.onedev.server.util.concurrent.BatchWorkManager;
import io.onedev.server.util.concurrent.BatchWorker;
import io.onedev.server.util.concurrent.Prioritized;
import io.onedev.server.util.patternset.PatternSet;
@Singleton
public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
@ -319,7 +349,8 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
}
private IndexResult doIndex(Project project, ObjectId commit) {
try (Directory directory = FSDirectory.open(projectManager.getIndexDir(project.getId()).toPath())) {
var indexDir = projectManager.getIndexDir(project.getId());
try (Directory directory = FSDirectory.open(indexDir.toPath())) {
if (DirectoryReader.indexExists(directory)) {
try (IndexReader reader = DirectoryReader.open(directory)) {
IndexSearcher searcher = new IndexSearcher(reader);
@ -327,6 +358,9 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
return new IndexResult(0, 0);
else
return doIndex(project, commit, directory, searcher);
} catch (IndexFormatTooOldException e) {
FileUtils.cleanDir(indexDir);
return doIndex(project, commit, directory, null);
}
} else {
return doIndex(project, commit, directory, null);
@ -364,6 +398,9 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
try (IndexReader reader = DirectoryReader.open(directory)) {
IndexSearcher searcher = new IndexSearcher(reader);
return getIndexVersion().equals(getCommitIndexVersion(searcher, commitId));
} catch (IndexFormatTooOldException e) {
FileUtils.cleanDir(indexDir);
return false;
}
} else {
return false;
@ -375,41 +412,6 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
});
}
@Sessional
@Listen
public void on(RefUpdated event) {
// only index branches at back end, tags will be indexed on demand from GUI
// as many tags might be pushed all at once when the repository is imported
if (event.getRefName().startsWith(Constants.R_HEADS)
&& !event.getNewCommitId().equals(ObjectId.zeroId())) {
IndexWork work = new IndexWork(BACKEND_PRIORITY, event.getNewCommitId());
batchWorkManager.submit(getBatchWorker(event.getProject().getId()), work);
}
}
@Sessional
@Listen
public void on(SystemStarted event) {
for (var file: projectManager.getProjectsDir().listFiles()) {
if (!NumberUtils.isDigits(file.getName()))
continue;
var projectId = Long.valueOf(file.getName());
var indexDir = projectManager.getIndexDir(projectId);
if (indexDir.exists()) {
try (var directory = FSDirectory.open(indexDir.toPath())) {
if (DirectoryReader.indexExists(directory)) {
try (var reader = DirectoryReader.open(directory)) {
} catch (IndexFormatTooOldException e) {
FileUtils.cleanDir(indexDir);
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
}
@Override
public void indexAsync(Long projectId, ObjectId commitId) {

62
server-core/src/main/java/io/onedev/server/search/entity/project/CommitDateCriteria.java
View File

@ -1,62 +0,0 @@
package io.onedev.server.search.entity.project;
import java.util.Date;
import javax.annotation.Nullable;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.From;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import io.onedev.server.model.Project;
import io.onedev.server.model.ProjectLastEventDate;
import io.onedev.server.search.entity.EntityQuery;
import io.onedev.server.util.ProjectScope;
import io.onedev.server.util.criteria.Criteria;
public class CommitDateCriteria extends Criteria<Project> {
private static final long serialVersionUID = 1L;
private final int operator;
private final String value;
private final Date date;
public CommitDateCriteria(String value, int operator) {
date = EntityQuery.getDateValue(value);
this.operator = operator;
this.value = value;
}
@Override
public Predicate getPredicate(@Nullable ProjectScope projectScope, CriteriaQuery<?> query, From<Project, Project> from, CriteriaBuilder builder) {
Path<Date> attribute = ProjectQuery.getPath(from, Project.PROP_LAST_EVENT_DATE + "." + ProjectLastEventDate.PROP_COMMIT);
if (operator == ProjectQueryLexer.IsUntil)
return builder.lessThan(attribute, date);
else
return builder.greaterThan(attribute, date);
}
@Override
public boolean matches(Project project) {
if (project.getLastEventDate().getCommit() != null) {
if (operator == ProjectQueryLexer.IsUntil)
return project.getLastEventDate().getCommit().before(date);
else
return project.getLastEventDate().getCommit().after(date);
} else {
return false;
}
}
@Override
public String toStringWithoutParens() {
return Criteria.quote(Project.NAME_LAST_COMMIT_DATE) + " "
+ ProjectQuery.getRuleName(operator) + " "
+ Criteria.quote(value);
}
}

8
server-core/src/main/java/io/onedev/server/search/entity/project/LastActivityDateCriteria.java
View File

@ -10,7 +10,7 @@ import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import io.onedev.server.model.Project;
import io.onedev.server.model.ProjectLastEventDate;
import io.onedev.server.model.ProjectLastActivityDate;
import io.onedev.server.search.entity.EntityQuery;
import io.onedev.server.util.ProjectScope;
import io.onedev.server.util.criteria.Criteria;
@ -41,7 +41,7 @@ public class LastActivityDateCriteria extends Criteria<Project> {
@Override
public Predicate getPredicate(@Nullable ProjectScope projectScope, CriteriaQuery<?> query, From<Project, Project> from, CriteriaBuilder builder) {
Path<Date> attribute = ProjectQuery.getPath(from, Project.PROP_LAST_EVENT_DATE + "." + ProjectLastEventDate.PROP_ACTIVITY);
Path<Date> attribute = ProjectQuery.getPath(from, Project.PROP_LAST_ACTIVITY_DATE + "." + ProjectLastActivityDate.PROP_VALUE);
if (operator == ProjectQueryLexer.IsUntil)
return builder.lessThan(attribute, date);
else
@ -51,9 +51,9 @@ public class LastActivityDateCriteria extends Criteria<Project> {
@Override
public boolean matches(Project project) {
if (operator == ProjectQueryLexer.IsUntil)
return project.getLastEventDate().getActivity().before(date);
return project.getLastActivityDate().getValue().before(date);
else
return project.getLastEventDate().getActivity().after(date);
return project.getLastActivityDate().getValue().after(date);
}
@Override

7
server-core/src/main/java/io/onedev/server/search/entity/project/ProjectQuery.java
View File

@ -182,10 +182,7 @@ public class ProjectQuery extends EntityQuery<Project> {
break;
case IsUntil:
case IsSince:
if (fieldName.equals(Project.NAME_LAST_ACTIVITY_DATE))
criterias.add(new LastActivityDateCriteria(value, operator));
else
criterias.add(new CommitDateCriteria(value, operator));
criterias.add(new LastActivityDateCriteria(value, operator));
break;
default:
throw new ExplicitException("Unexpected operator " + getRuleName(operator));
@ -272,7 +269,7 @@ public class ProjectQuery extends EntityQuery<Project> {
case IsUntil:
case IsSince:
if (!fieldName.equals(Project.NAME_LAST_ACTIVITY_DATE)
&& !fieldName.equals(Project.NAME_LAST_COMMIT_DATE)) {
&& !fieldName.equals(Project.NAME_LAST_ACTIVITY_DATE)) {
throw newOperatorException(fieldName, operator);
}
break;

2
server-core/src/main/java/io/onedev/server/web/behavior/ProjectQueryBehavior.java
View File

@ -87,7 +87,7 @@ public class ProjectQueryBehavior extends ANTLRAssistBehavior {
String fieldName = ProjectQuery.getValue(fieldElements.get(0).getMatchedText());
try {
ProjectQuery.checkField(fieldName, operator);
if (fieldName.equals(Project.NAME_LAST_ACTIVITY_DATE) || fieldName.equals(Project.NAME_LAST_COMMIT_DATE)) {
if (fieldName.equals(Project.NAME_LAST_ACTIVITY_DATE)) {
List<InputSuggestion> suggestions = SuggestionUtils.suggest(DateUtils.RELAX_DATE_EXAMPLES, matchWith);
return !suggestions.isEmpty()? suggestions: null;
} else if (fieldName.equals(Project.NAME_NAME)) {

265
server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java
View File

@ -23,7 +23,6 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.Stack;
@ -31,6 +30,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.inject.Inject;
@ -57,8 +57,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.Sets;
import com.hazelcast.map.EntryProcessor;
import com.hazelcast.map.IMap;
import io.onedev.commons.loader.ManagedSerializedForm;
import io.onedev.commons.utils.ExplicitException;
@ -81,7 +79,6 @@ import io.onedev.server.event.project.ActiveServerChanged;
import io.onedev.server.event.project.RefUpdated;
import io.onedev.server.event.project.issue.IssueCommitsAttached;
import io.onedev.server.event.system.SystemStarted;
import io.onedev.server.event.system.SystemStarting;
import io.onedev.server.git.GitContribution;
import io.onedev.server.git.GitContributor;
import io.onedev.server.git.GitUtils;
@ -211,9 +208,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
private final Map<Long, Integer> totalCommitCountCache = new ConcurrentHashMap<>();
private final Map<Long, List<NameAndEmail>> usersCache = new ConcurrentHashMap<>();
private volatile IMap<String, Set<Long>> contributedProjects;
private final Map<Long, Pair<Set<NameAndEmail>, Set<String>>> usersCache = new ConcurrentHashMap<>();
@Inject
public DefaultCommitInfoManager(ProjectManager projectManager,
@ -257,6 +252,19 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
Repository repository = projectManager.getRepository(project.getId());
Pair<byte[], ObjectId> result = env.computeInTransaction(txn -> {
var users = usersCache.get(project.getId());
if (users == null) {
byte[] userBytes = readBytes(defaultStore, txn, USERS_KEY);
if (userBytes != null) {
Set<NameAndEmail> nameAndEmails = SerializationUtils.deserialize(userBytes);
users = new Pair<>(nameAndEmails,
nameAndEmails.stream().map(NameAndEmail::getEmailAddress).collect(Collectors.toSet()));
} else {
users = new Pair<>(new HashSet<>(), new HashSet<>());
}
usersCache.put(project.getId(), users);
}
ByteIterable commitKey = new CommitByteIterable(commitId);
byte[] commitBytes = readBytes(commitsStore, txn, commitKey);
@ -288,12 +296,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
Map<Long, Integer> commitCountCache = new HashMap<>();
Set<NameAndEmail> users;
byte[] userBytes = readBytes(defaultStore, txn, USERS_KEY);
if (userBytes != null)
users = SerializationUtils.deserialize(userBytes);
else
users = new HashSet<>();
var users = usersCache.get(project.getId());
var usersChanged = new AtomicBoolean(false);
new ElementPumper<LogCommit>() {
@ -384,14 +387,16 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
}
if (currentCommit.getCommitter() != null) {
if (users.add(new NameAndEmail(currentCommit.getCommitter())))
if (users.getLeft().add(new NameAndEmail(currentCommit.getCommitter())))
usersChanged.set(true);
users.getRight().add(currentCommit.getCommitter().getEmailAddress());
}
if (currentCommit.getAuthor() != null) {
NameAndEmail nameAndEmail = new NameAndEmail(currentCommit.getAuthor());
if (users.add(nameAndEmail))
if (users.getLeft().add(nameAndEmail))
usersChanged.set(true);
users.getRight().add(nameAndEmail.getEmailAddress());
ByteIterable authorKey = new ArrayByteIterable(SerializationUtils.serialize(nameAndEmail));
int userIndex = readInt(userToIndexStore, txn, authorKey, -1);
@ -475,10 +480,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
writeInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, nextIndex.path);
if (usersChanged.get()) {
userBytes = SerializationUtils.serialize((Serializable) users);
var userBytes = SerializationUtils.serialize((Serializable) users.getLeft());
defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));
usersCache.remove(project.getId());
updateContributedProjects(project.getId(), users);
}
for (Map.Entry<Long, Integer> entry : commitCountCache.entrySet())
@ -497,26 +500,6 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
logger.debug("Collected commit information (project: {}, ref: {})", project.getPath(), refName);
}
private void updateContributedProjects(Long projectId, Set<NameAndEmail> users) {
Set<String> emailAddresses = users.stream()
.map(NameAndEmail::getEmailAddress)
.collect(toSet());
contributedProjects.executeOnKeys(emailAddresses, new EntryProcessor<String, Set<Long>, Object>() {
@Override
public Object process(IMap.Entry<String, Set<Long>> entry) {
Set<Long> contributedProjectsOfEmail = entry.getValue();
if (contributedProjectsOfEmail == null) {
contributedProjectsOfEmail = new HashSet<>();
}
if (contributedProjectsOfEmail.add(projectId)) {
entry.setValue(contributedProjectsOfEmail);
}
return null;
}
});
}
private void collectContributions(Project project, ObjectId commitId) {
Environment env = getEnv(project.getId().toString());
Store defaultStore = getStore(env, DEFAULT_STORE);
@ -908,25 +891,14 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
@Override
public List<NameAndEmail> call() {
List<NameAndEmail> users = usersCache.get(projectId);
if (users == null) {
Environment env = getEnv(projectId.toString());
Store store = getStore(env, DEFAULT_STORE);
users = env.computeInReadonlyTransaction(txn -> {
byte[] bytes = readBytes(store, txn, USERS_KEY);
if (bytes != null) {
List<NameAndEmail> innerUsers =
new ArrayList<>(SerializationUtils.deserialize(bytes));
Collections.sort(innerUsers);
return innerUsers;
} else {
return new ArrayList<>();
}
});
usersCache.put(projectId, users);
var users = usersCache.get(projectId);
if (users != null) {
var sortedUsers = new ArrayList<>(users.getLeft());
Collections.sort(sortedUsers);
return sortedUsers;
} else {
return new ArrayList<>();
}
return users;
}
});
@ -1112,8 +1084,37 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
sessionManager.run(() -> {
Project project = projectManager.load(projectId);
List<CollectingWork> collectingWorks = new ArrayList<>();
for (Object work : works)
collectingWorks.add((CollectingWork) work);
for (Object work : works) {
if (work instanceof CollectingWork) {
collectingWorks.add((CollectingWork) work);
} else if (work instanceof CheckingWork) {
try (RevWalk revWalk = new RevWalk(projectManager.getRepository(projectId))) {
Collection<Ref> refs = new ArrayList<>();
refs.addAll(projectManager.getRepository(projectId).getRefDatabase()
.getRefsByPrefix(Constants.R_HEADS));
refs.addAll(projectManager.getRepository(projectId).getRefDatabase()
.getRefsByPrefix(Constants.R_TAGS));
for (Ref ref : refs) {
RevObject revObj;
try {
revObj = revWalk.peel(revWalk.parseAny(ref.getObjectId()));
} catch (MissingObjectException e) {
var message = String.format("%s (project id: %d, ref: %s)", e.getMessage(),
projectId, ref.getName());
throw new ExplicitException(message);
}
if (revObj instanceof RevCommit) {
RevCommit commit = (RevCommit) revObj;
collectingWorks.add(new CollectingWork(CHECK_PRIORITY, commit.copy(),
commit.getCommitTime(), ref.getName()));
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
collectingWorks.sort(new CommitTimeComparator());
for (CollectingWork work : collectingWorks)
@ -1124,65 +1125,12 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
};
}
private boolean collect(Long projectId, int priority) {
List<CollectingWork> works = new ArrayList<>();
try (RevWalk revWalk = new RevWalk(projectManager.getRepository(projectId))) {
Collection<Ref> refs = new ArrayList<>();
refs.addAll(projectManager.getRepository(projectId).getRefDatabase().getRefsByPrefix(Constants.R_HEADS));
refs.addAll(projectManager.getRepository(projectId).getRefDatabase().getRefsByPrefix(Constants.R_TAGS));
for (Ref ref : refs) {
RevObject revObj;
try {
revObj = revWalk.peel(revWalk.parseAny(ref.getObjectId()));
} catch (MissingObjectException e) {
var message = String.format("%s (project id: %d, ref: %s)", e.getMessage(), projectId, ref.getName());
throw new ExplicitException(message);
}
if (revObj instanceof RevCommit) {
RevCommit commit = (RevCommit) revObj;
works.add(new CollectingWork(priority, commit.copy(),
commit.getCommitTime(), ref.getName()));
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
if (!works.isEmpty()) {
works.sort(new CommitTimeComparator());
for (CollectingWork work : works)
batchWorkManager.submit(getBatchWorker(projectId), work);
return true;
} else {
return false;
}
}
@Sessional
@Listen
public void on(SystemStarting event) {
contributedProjects = clusterManager.getHazelcastInstance().getMap("contributedProjects");
}
@Sessional
@Listen
public void on(SystemStarted event) {
logger.info("Caching code contribution info...");
for (var projectId: projectManager.getActiveIds()) {
checkVersion(getEnvDir(projectId.toString()));
if (collect(projectId, CHECK_PRIORITY)) {
Environment env = getEnv(projectId.toString());
Store store = getStore(env, DEFAULT_STORE);
env.computeInReadonlyTransaction(txn -> {
byte[] bytes = readBytes(store, txn, USERS_KEY);
if (bytes != null)
updateContributedProjects(projectId, SerializationUtils.deserialize(bytes));
return null;
});
}
batchWorkManager.submit(getBatchWorker(projectId), new CheckingWork(CHECK_PRIORITY));
}
}
@ -1191,7 +1139,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
public void on(ActiveServerChanged event) {
for (var projectId: event.getProjectIds()) {
checkVersion(getEnvDir(projectId.toString()));
collect(projectId, CHECK_PRIORITY);
batchWorkManager.submit(getBatchWorker(projectId), new CheckingWork(CHECK_PRIORITY));
}
}
@ -1258,6 +1206,14 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
});
}
static class CheckingWork extends Prioritized {
public CheckingWork(int priority) {
super(priority);
}
}
static class CollectingWork extends Prioritized {
private final String refName;
@ -1564,53 +1520,54 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
@Sessional
@Override
public Map<Long, Map<ObjectId, Long>> getUserCommits(User user, Date fromDate, Date toDate) {
var emailAddresses = user.getEmailAddresses().stream()
.filter(it -> it.isVerified())
.map(it -> it.getValue())
.collect(toSet());
var userCommits = new HashMap<Long, Map<ObjectId, Long>>();
if (contributedProjects != null) {
var emailAddresses = user.getEmailAddresses().stream()
.filter(it -> it.isVerified())
.map(it -> it.getValue())
.collect(toSet());
var cache = projectManager.cloneCache();
var projectIds = new HashSet<Long>();
contributedProjects.getAll(emailAddresses).values().stream()
.filter(Objects::nonNull)
.forEach(projectIds::addAll);
for (var projectId: projectIds) {
var project = cache.get(projectId);
if (project != null && project.isCodeManagement() && project.getForkedFromId() == null) {
var activeServer = projectManager.getActiveServer(projectId, false);
if (activeServer != null) {
userCommits.put(projectId, clusterManager.runOnServer(activeServer, new ClusterTask<>() {
Map<String, Map<Long, Map<ObjectId, Long>>> result = clusterManager.runOnAllServers(new ClusterTask<>() {
private static final long serialVersionUID = 1L;
@Override
public Map<ObjectId, Long> call() {
Environment env = getEnv(projectId.toString());
Store userCommitsStore = getStore(env, USER_COMMITS_STORE);
return env.computeInReadonlyTransaction(new TransactionalComputable<Map<ObjectId, Long>>() {
@Override
public Map<ObjectId, Long> compute(Transaction txn) {
var userCommits = new HashMap<ObjectId, Long>();
for (var emailAddress: emailAddresses) {
deserializeUserCommits(readBytes(userCommitsStore, txn, new StringByteIterable(emailAddress))).entrySet().forEach(it -> {
if (it.getValue() >= fromDate.getTime() && it.getValue() <= toDate.getTime())
userCommits.put(it.getKey(), it.getValue());
});
}
return userCommits;
@Override
public Map<Long, Map<ObjectId, Long>> call() {
var localServer = clusterManager.getLocalServerAddress();
var userCommits = new HashMap<Long, Map<ObjectId, Long>>();
for (var entry: usersCache.entrySet()) {
var projectId = entry.getKey();
if (localServer.equals(projectManager.getActiveServer(projectId, false))
&& emailAddresses.stream().anyMatch(entry.getValue().getRight()::contains)) {
var project = projectManager.findFacade(projectId);
if (project != null && project.isCodeManagement() && project.getForkedFromId() == null) {
Environment env = getEnv(projectId.toString());
Store userCommitsStore = getStore(env, USER_COMMITS_STORE);
userCommits.put(projectId, env.computeInReadonlyTransaction(new TransactionalComputable<Map<ObjectId, Long>>() {
@Override
public Map<ObjectId, Long> compute(Transaction txn) {
var userCommits = new HashMap<ObjectId, Long>();
for (var emailAddress : emailAddresses) {
deserializeUserCommits(
readBytes(userCommitsStore, txn, new StringByteIterable(emailAddress)))
.entrySet().forEach(it -> {
if (it.getValue() >= fromDate.getTime()
&& it.getValue() <= toDate.getTime())
userCommits.put(it.getKey(), it.getValue());
});
}
});
}
}));
return userCommits;
}
}));
}
}
}
return userCommits;
}
}
});
for (var entry: result.entrySet())
userCommits.putAll(entry.getValue());
return userCommits;
}

6
server-product/src/main/java/io/onedev/server/product/ProductServletConfigurator.java
View File

@ -167,9 +167,9 @@ public class ProductServletConfigurator implements ServletConfigurator {
context.addServlet(new ServletHolder(jerseyServlet), "/~api/*");
context.addServlet(new ServletHolder(serverServlet), "/~server");
var mcpServletHolder = new ServletHolder(mcpServerServlet);
context.addServlet(mcpServletHolder, "/~mcp");
context.addServlet(mcpServletHolder, "/~mcp/*");
//var mcpServletHolder = new ServletHolder(mcpServerServlet);
//context.addServlet(mcpServletHolder, "/~mcp");
//context.addServlet(mcpServletHolder, "/~mcp/*");
}
}