Securely connecting
siloed content to AI pipelines.

# Scaffold a ready-to-build custom connector with tests, Docker Compose overlay & Admin UI integration
# Available archetypes under org.opencrawling.archetypes:
#   1. opencrawling-archetype-repository-connector      (Ingestion sources: CMS, APIs, DBs)
#   2. opencrawling-archetype-output-connector          (Vector stores, Search engines, Sinks)
#   3. opencrawling-archetype-transformation-connector  (Custom chunking, enrichment, redaction)

mvn archetype:generate \
  -DarchetypeGroupId=org.opencrawling.archetypes \
  -DarchetypeArtifactId=opencrawling-archetype-repository-connector \
  -DarchetypeVersion=1.0.4 \
  -DgroupId=com.mycompany.connectors \
  -DartifactId=my-repository-connector \
  -Dversion=1.0.4 \
  -DconnectorName=SampleRepositoryConnector

# Build, run unit & Failsafe integration tests (*Test.java & *IT.java), and launch with Docker overlay:
cd my-repository-connector
mvn clean package
docker compose -f docker/docker-compose.dist.yml -f docker/docker-compose.override.yml up -d
mvn verify

<!-- Add core connector API to your Spring Boot application -->
<dependency>
    <groupId>org.opencrawling</groupId>
    <artifactId>oc-core</artifactId>
    <version>1.0.4</version>
</dependency>

<!-- Include FileSystem Repository Connector if required -->
<dependency>
    <groupId>org.opencrawling</groupId>
    <artifactId>oc-filesystem-repository-connector</artifactId>
    <version>1.0.4</version>
</dependency>

// Tip: Scaffold this project automatically using the Maven Archetypes tab!
// Implements RepositoryConnector (or OutputConnector / TransformationConnector) SPI
@Component
public class SampleRepositoryConnector implements RepositoryConnector {

    private final SampleRepositoryConnectorConfiguration configuration;
    private final SampleRepositoryConnectorFetcher fetcher;
    private boolean connected = false;

    @Override
    public String getName() {
        return "SampleRepositoryConnector";
    }

    @Override
    public void connect() throws Exception {
        // Initialize authenticated client session with target repository
        this.connected = true;
    }

    @Override
    public Flux<RepositoryDocument> scan(String basePath) {
        // Stream documents reactively with metadata and Zero-Trust ACLs
        return fetcher.fetchDocuments(basePath);
    }

    @Override
    public ConnectorSchema getSchema(String basePath) {
        return new ConnectorSchema(List.of());
    }
}