From b6157c67ae7fb734508ec4564b5e1ed5e82d2681 Mon Sep 17 00:00:00 2001 From: valentinkolb Date: Thu, 26 Oct 2023 22:34:59 +0200 Subject: [PATCH] added plugins --- .gitignore | 15 +++ Readme.md | 189 ++++++++++++++++++++++++++++++++ go.mod | 94 ++++++++++++++++ ldapLogin/main.go | 71 ++++++++++++ ldapSync/db.go | 143 +++++++++++++++++++++++++ ldapSync/ldapSync.go | 244 +++++++++++++++++++++++++++++++++++++++++ ldapSync/main.go | 102 ++++++++++++++++++ ldapSync/models.go | 24 +++++ ldapSync/tables.go | 250 +++++++++++++++++++++++++++++++++++++++++++ logger/main.go | 47 ++++++++ main/main.go | 35 ++++++ 11 files changed, 1214 insertions(+) create mode 100644 .gitignore create mode 100644 Readme.md create mode 100644 go.mod create mode 100644 ldapLogin/main.go create mode 100644 ldapSync/db.go create mode 100644 ldapSync/ldapSync.go create mode 100644 ldapSync/main.go create mode 100644 ldapSync/models.go create mode 100644 ldapSync/tables.go create mode 100644 logger/main.go create mode 100644 main/main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fd94a56 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# Webstorm +.idea + +# js +node_modules +dist + +# pocketbase +pb_data + +# env +*.env.local + +# macOS +.DS_Store \ No newline at end of file diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..a3ae65f --- /dev/null +++ b/Readme.md @@ -0,0 +1,189 @@ +# StuVe IT Tools Backend + +## Übersicht + +Als Backend für die StuVe IT Tools wird [Pocketbase](https://pocketbase.io) +verwendet. Pocketbase ist ein Backend-as-a-Service, der auf SQLite basiert. + +## Was ist Pocketbase? + +Implementiert ist Pocketbase in Golang und kann dort auch einfach als +Library eingebunden werden und so [erweitert](https://pocketbase.io/docs/go-overview/) werden. +Dies wurde in diesem Projekt auch gemacht, um die Authentifizierung +um eine LDAP Authentifizierung zu erweitern. + +Aus diesem Grund handelt es sich bei diesem Projekt nicht um +einen Fork von Pocketbase, sondern um eine Library, die Pocketbase +erweitert. + +Pocketbase selbst bietet folgende Features: + +- Datenbank + - Als Datenbank verwendet Pocketbase SQLite +- Authentifizierung + - Pocketbase bietet sogenannte "Auth-Collections" an, die eine + Authentifizierung über E-Mail und Passwort ermöglichen. + - Es ist allerdings möglich dieses Mechanismus zu erweitern was in + diesem Projekt auch gemacht wurde. (siehe unten) +- File Storage + - Pocketbase bietet einen File Storage an, der allerdings nur + Dateien bis 5B unterstützt. + - Achtung: Per Default sind alle Dateien öffentlich zugänglich. +- E-Mail Versand (limitiert aber in Golang erweiterbar) + - Pocketbase sendet per Default nur System E-Mails, zb Passwort + vergessen. + - Über Golang ist es allerdings möglich den E-Mailversand zu + erweitern. Dies ist in **Planung**. +- Realtime Events (Websockets) + - Pocketbase bietet einen Realtime Event Service an, der über + Websockets funktioniert. Dabei können auf alle Datenbank Events (Insert, + Update, Delete) reagiert werden. +- JavaScript SDK + - Pocketbase bietet ein JavaScript SDK an, das alle Features von + Pocketbase unterstützt. + +## Entwickeln + +todo + +## Installation + +todo + +## Projekt Struktur + +Die einzelnen custom go Packages sind weiter unten beschrieben. + +``` +├── Dockefile # Dockerfile für die Entwicklung +├── .dockerignore # Dateien die nicht in den Docker Container kopiert werden sollen +├── .gitignore # Dateien die nicht in Git gepusht werden sollen +├── go.mod # Go Module Datei (Go's Package Manager) +├── main # Main Package (App Entry Point) +│ └── main.go # Main Funktion +├── logger # Custom Logger Package +│ └── main.go # Logger Funktionen +├── ldapSync # LDAP Sync Package +│ ├── main.go # LDAP Sync Entry Point +│ ├── db.go # Datenbank Funktionen (Insert Users, Groups, ...) +│ ├── models.go # Structs für LDAP User und Group +│ ├── tables.go # Erstellt die benötigten Tabellen in der Datenbank (Users, Groups, Logs) +│ └── ldapSync.go # Sync Funktionen (Sync Users, Groups, ...) +└── ldapLogin # LDAP Login Package + └── main.go # Login Funktionen (Login Route) +``` + +## Custom Packages + +Damit Pocketbase mit der StuVe IT Tools Infrastruktur funktioniert, +wurden einige Packages erstellt, die Pocketbase erweitern. + +### Logger + +Das Logger Package ist ein custom Logger, der die Log-Nachrichten +farblich in die Konsole schreibt. +Diese Package ist sehr minimal gehalten und bietet nur die +Funktionen `Info`, `Warn` und `Error` an. + +### LDAP Sync + +Dieses Package ist das umfangreichste Package und synchronisiert +die LDAP Datenbank mit der Pocketbase Datenbank. + +Dabei werden alle LDAP User und Gruppen in die Pocketbase Datenbank +kopiert. + +#### Konfiguration + +Das Package benötigt folgende ENV Variablen: + +- `LDAP_URL`: LDAP Server URL (e.g. `ldap://ldap.stuve.uni-ulm.de`) +- `LDAP_BIND_DN`: LDAP Bind DN (zum Lesen der LDAP Datenbank, admin user) +- `LDAP_BIND_PASSWORD`: LDAP Bind Password (das Passwort des admin users) +- `LDAP_USER_BASE_DN`: LDAP User Base DN (hier werden alle User gesucht, + e.g. `ou=useraccounts,ou=user,dc=stuve,dc=uni-ulm,dc=de`) +- `LDAP_USER_FILTER`: LDAP User Filter (e.g. `(|(objectCategory=person)(objectClass=user))`) +- `LDAP_GROUP_BASE_DN`: LDAP Group Base DN (hier werden alle Gruppen gesucht, + e.g. `ou=groups,ou=user,dc=stuve,dc=uni-ulm,dc=de`) +- `LDAP_GROUP_FILTER`: LDAP Group Filter (e.g. `(objectClass=group)`) +- `LDAP_SUNC_SCHEDULE`: LDAP Sync Schedule (e.g. `*/1 * * * *` für minütliches Syncen) + (siehe [Cron](https://en.wikipedia.org/wiki/Cron)) + +#### Tabellen + +Das Package erstellt folgende Tabellen in der Pocketbase Datenbank: + +- `ldap_users`: Alle LDAP User ([auth-Collection](https://pocketbase.io/docs/collections/#auth-collection)) +- `ldap_groups`: Alle LDAP Gruppen +- `ldap_sync_logs`: Alle LDAP Sync Logs (wann wurde was synchronisiert) + +Die Tabellen werden automatisch erstellt, wenn das Programm gestartet wird und sie noch nicht existieren. + +#### Sync + +Das Package synchronisiert die LDAP Datenbank mit der Pocketbase Datenbank. + +Dabei werden alle LDAP User und Gruppen in die Pocketbase Datenbank +kopiert. Es werden `memberOf` ldap Attribute verwendet, um die +Gruppenzugehörigkeit zu bestimmen und in SQL Relationen umzuwandeln. + +Beim ersten synchronisieren kann es sein, dass eine `memberOf` Gruppe +noch nicht existiert. In diesem Fall wird die Beziehung erst beim +nächsten Sync erstellt. Falls eine `memberOf` Gruppe nicht gefunden wird, +wird dies in der Konsole geloggt. + +Für jede Synchronisation wird ein Log in der `ldap_sync_logs` Tabelle +erstellt. + +Sowohl für die ldap_users als auch für die ldap_groups Tabelle wird +unter den API Rules das bearbeiten und löschen deaktiviert, da diese +Tabellen nur synchronisiert werden und nicht manuell bearbeitet werden +sollen. +Dadruch können sie nur über das LDAP Sync Package (und die Admin UI) bearbeitet werden. + +#### User Sync + +Damit man sich später auch als Nutzer anmelden kann, werden die +User in einer `auth` Collection gespeichert. Allerdings werden +natürlich nicht die LDAP Passwörter synchronisiert. + +Aus diesem Grund muss auch jede Auth-Methode unter Options im Admin UI +für diese Collection deaktiviert werden (Username, E-Mail, OAuth2). + +Mehr dazu unter dem LDAP Login Package. + +### LDAP Login + +Dieses Package ist für die Authentifizierung zuständig. +Es implementiert eine LDAP Authentifizierung, die Pocketbase +um eine LDAP Authentifizierung erweitert. + +#### Konfiguration + +Das Package benötigt folgende ENV Variablen: + +- `LDAP_URL`: LDAP Server URL (e.g. `ldap://ldap.stuve.uni-ulm.de`) + +#### Mechanismus + +Dieses Package für der Api die folgende Route hinzu: + +`GET /api/ldap/login` + +Diese Route nimmt die Parameter `username` und `password` als json Body entgegen: + +```json +{ + "username": "username", + "password": "password" +} +``` + +Dabei wird das Passwort mit dem LDAP Server abgeglichen und falls +erfolgreich ein Token und das ldap_user Modell zurückgegeben. + +Diese Response ist identisch mit der standard Pocketbase Authentifizierung. + +#### Token Refresh + +todo diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f372ec5 --- /dev/null +++ b/go.mod @@ -0,0 +1,94 @@ +module gitlab.uni-ulm.de/stuve-it/it-tools/backend + +go 1.21 + +require ( + github.com/go-ldap/ldap/v3 v3.4.6 + github.com/pocketbase/pocketbase v0.19.0 +) + +require ( + github.com/AlecAivazis/survey/v2 v2.3.7 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go v1.45.25 // indirect + github.com/aws/aws-sdk-go-v2 v1.21.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.45 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.43 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect + github.com/aws/smithy-go v1.15.0 // indirect + github.com/disintegration/imaging v1.6.2 // indirect + github.com/domodwyer/mailyak/v3 v3.6.2 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/ganigeorgiev/fexpr v0.3.0 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect + github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/google/wire v0.5.0 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-sqlite3 v1.14.17 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/pocketbase/dbx v1.10.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v1.2.2 // indirect + go.opencensus.io v0.24.0 // indirect + gocloud.dev v0.34.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/image v0.13.0 // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.14.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/api v0.147.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/grpc v1.58.3 // indirect + google.golang.org/protobuf v1.31.0 // indirect + lukechampine.com/uint128 v1.3.0 // indirect + modernc.org/cc/v3 v3.41.0 // indirect + modernc.org/ccgo/v3 v3.16.15 // indirect + modernc.org/libc v1.24.1 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/sqlite v1.26.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect +) diff --git a/ldapLogin/main.go b/ldapLogin/main.go new file mode 100644 index 0000000..7c7d98a --- /dev/null +++ b/ldapLogin/main.go @@ -0,0 +1,71 @@ +package ldapLogin + +import ( + "fmt" + "github.com/go-ldap/ldap/v3" + "github.com/labstack/echo/v5" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/apis" + "github.com/pocketbase/pocketbase/core" + "gitlab.uni-ulm.de/stuve-it/it-tools/backend/ldapSync" + "os" +) + +// InitLDAPLogin initializes ldap login endpoint +// +// this endpoint is used to authenticate users against ldap server +// it adds the following endpoint to the app: +// GET /api/ldap/login +// +// the endpoint expects the following request data: +// +// { +// "cn": "user common name", +// "password": "user password" +// } +// +// if the user is authenticated successfully the endpoint returns and apis.RecordAuthResponse +func InitLDAPLogin(app *pocketbase.PocketBase, e *core.ServeEvent) error { + + e.Router.GET("/api/ldap/login", func(c echo.Context) error { + + // step 1: get data from request + data := struct { + CN string `json:"cn" form:"cn"` + Password string `json:"password" form:"password"` + }{} + if err := c.Bind(&data); err != nil { + return apis.NewBadRequestError("Failed to read request data", err) + } + + // step 2: get ldap user by cn from ldapUsers table + record, err := ldapSync.GetLdapUserByCN(app, data.CN) + + // if user does not exist in ldapUsers table return error + if err != nil { + return apis.NewBadRequestError("Invalid credentials", err) + } + + // step 3: connect to ldap server + conn, err := ldap.DialURL(os.Getenv("LDAP_URL")) + if err != nil { + return apis.NewBadRequestError( + "Failed to read request data", + fmt.Errorf("unable to connect to ldap server - %s", err), + ) + } + defer conn.Close() + + // step 4: bind to ldap server with user credentials from request + err = conn.Bind(data.CN, data.Password) + if err != nil { + // if bind fails return error - invalid credentials + return apis.NewBadRequestError("Invalid credentials", err) + } + + // return auth response + return apis.RecordAuthResponse(app, c, record, nil) + }, apis.ActivityLogger(app)) + + return nil +} diff --git a/ldapSync/db.go b/ldapSync/db.go new file mode 100644 index 0000000..f0f4f5b --- /dev/null +++ b/ldapSync/db.go @@ -0,0 +1,143 @@ +package ldapSync + +/* +this file contains functions to sync ldap users and groups to the database +*/ + +import ( + "fmt" + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/forms" + "github.com/pocketbase/pocketbase/models" + "gitlab.uni-ulm.de/stuve-it/it-tools/backend/logger" +) + +// upsertLDAPGroup This function creates / updates a record in the ldap groups table +func upsertLDAPGroup(app *pocketbase.PocketBase, ldapGroup *LDAPGroup) error { + + // find ldapGroups table + collection, err := app.Dao().FindCollectionByNameOrId(ldapGroupsTableName) + if err != nil { + return err + } + + var record *models.Record + + // if record exists, update it + if res, _ := app.Dao().FindFirstRecordByFilter( + ldapGroupsTableName, + "gidNumber = {:gidNumber}", + dbx.Params{"gidNumber": ldapGroup.gidNumber}, + ); res != nil { + record = res + } else { // if record does not exist, create it + record = models.NewRecord(collection) + } + + form := forms.NewRecordUpsert(app, record) + + var groups []string + + // get group ids from group dns + for _, groupDn := range ldapGroup.memberOf { + group, err := GetLdapGroupByDN(app, groupDn) + if err == nil { + groups = append(groups, group.Id) + } else { + logger.LogErrorF("unable to find %s.memberOf: %s", ldapGroup.cn, groupDn) + } + } + + // load data + err = form.LoadData(map[string]any{ + "gidNumber": ldapGroup.gidNumber, + "cn": ldapGroup.cn, + "dn": ldapGroup.dn, + "description": ldapGroup.description, + "memberOf": groups, + }) + if err != nil { + return err + } + + // validate and submit (internally it calls app.Dao().SaveRecord(record) in a transaction) + if err := form.Submit(); err != nil { + return fmt.Errorf("failed to upsert group with dn: %s - %w", ldapGroup.dn, err) + } + + return nil +} + +// upsertLDAPUser This function creates / updates a record in the ldap users table +func upsertLDAPUser(app *pocketbase.PocketBase, ldapUser *LDAPUser) error { + + // find ldapUsers table + collection, err := app.Dao().FindCollectionByNameOrId(ldapUsersTableName) + if err != nil { + return err + } + + var record *models.Record + + // if record exists, update it + if res, _ := app.Dao().FindFirstRecordByFilter( + ldapUsersTableName, + "uidNumber = {:uidNumber}", + dbx.Params{"uidNumber": ldapUser.uidNumber}, + ); res != nil { + record = res + } else { // if record does not exist, create it + record = models.NewRecord(collection) + } + + accountExpires, _ := ldapTimeToUnixTime(ldapUser.accountExpires) + + var groups []string + + // get group ids from group dns + for _, groupDn := range ldapUser.memberOf { + group, err := GetLdapGroupByDN(app, groupDn) + if err == nil { + groups = append(groups, group.Id) + } else { + logger.LogErrorF("unable to find %s.memberOf: %s", ldapUser.cn, groupDn) + } + } + + record.Set("uidNumber", ldapUser.uidNumber) + record.Set("givenName", ldapUser.givenName) + record.Set("sn", ldapUser.sn) + record.Set("username", ldapUser.cn) + record.Set("accountExpires", accountExpires) + record.Set("email", ldapUser.mail) + record.Set("emailVisibility", false) + record.Set("verified", true) + record.Set("dn", ldapUser.dn) + record.Set("cn", ldapUser.cn) + record.Set("memberOf", groups) + + record.RefreshTokenKey() + + if err := app.Dao().SaveRecord(record); err != nil { + return fmt.Errorf("failed to upsert user with dn: %s - %w", ldapUser.dn, err) + } + + return nil +} + +func GetLdapGroupByDN(app *pocketbase.PocketBase, dn string) (*models.Record, error) { + return app.Dao().FindFirstRecordByFilter( + ldapGroupsTableName, + "dn = {:dn}", + dbx.Params{"dn": dn}, + ) +} + +func GetLdapUserByCN(app *pocketbase.PocketBase, cn string) (*models.Record, error) { + return app.Dao().FindFirstRecordByFilter( + ldapUsersTableName, + "cn = {:cn}", + dbx.Params{"cn": cn}, + ) +} diff --git a/ldapSync/ldapSync.go b/ldapSync/ldapSync.go new file mode 100644 index 0000000..f89772e --- /dev/null +++ b/ldapSync/ldapSync.go @@ -0,0 +1,244 @@ +package ldapSync + +import ( + "encoding/json" + "fmt" + "github.com/go-ldap/ldap/v3" + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/forms" + "github.com/pocketbase/pocketbase/models" + "gitlab.uni-ulm.de/stuve-it/it-tools/backend/logger" + "os" + "time" +) + +type SyncResult struct { + Found int + Synced int + Removed int + Errors []error +} + +// upsertLDAPGroup This function creates / updates a record in the ldap groups table +// +// this function expects that the table ldapGroups already exists +// +// old groups are removed from the database. a group is considered old if it has not been synced for 1 day +func syncLdapGroups(app *pocketbase.PocketBase, ldapClient *ldap.Conn) SyncResult { + var syncedCount int + var errors []error + + // Create a search request for groups + groupsSearchRequest := ldap.NewSearchRequest( + os.Getenv("LDAP_GROUP_BASE_DN"), // The base dn to search + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + os.Getenv("LDAP_GROUP_FILTER"), // The filter to apply + []string{"gidNumber", "description", "dn", "cn", "msSFU30NisDomain", "memberOf"}, + nil, + ) + + groupsFoundInLdap, err := ldapClient.Search(groupsSearchRequest) + if err != nil { + return SyncResult{ + Errors: []error{fmt.Errorf("unable to search in ldap for groups - %s", err)}, + } + } + + for _, entry := range groupsFoundInLdap.Entries { + err := upsertLDAPGroup(app, &LDAPGroup{ + gidNumber: entry.GetAttributeValue("gidNumber"), + description: entry.GetAttributeValue("description"), + dn: entry.DN, + cn: entry.GetAttributeValue("cn"), + msSFU30NisDomain: entry.GetAttributeValue("msSFU30NisDomain"), + memberOf: entry.GetAttributeValues("memberOf"), + }) + if err != nil { + errors = append(errors, err) + } else { + syncedCount++ + } + } + + var removedCount int + + // remove old groups + // step1: get a timeStamp one day ago + timeStamp := time.Now().AddDate(0, 0, -1) + + // step2: get all groups that have not been synced since that timeStamp + records, err := app.Dao().FindRecordsByFilter( + ldapGroupsTableName, + "updated < {:timeStamp}", "", 0, 0, + dbx.Params{"timeStamp": timeStamp}, + ) + + if err != nil { + errors = append(errors, fmt.Errorf("unable to get old ldap groups from db - %s", err)) + return SyncResult{ + Found: len(groupsFoundInLdap.Entries), + Synced: syncedCount, + Errors: errors, + } + } + + // step3: delete all groups that have not been synced since that timeStamp + for _, record := range records { + err := app.Dao().DeleteRecord(record) + if err != nil { + errors = append(errors, fmt.Errorf("unable to remove old group with dn: %s - %s", record.Get("dn"), err)) + } else { + removedCount++ + } + } + + return SyncResult{ + Found: len(groupsFoundInLdap.Entries), + Synced: syncedCount, + Removed: removedCount, + Errors: errors, + } +} + +// syncLdapUsers This function syncs the ldap users with the database +// +// this function expects that the tables ldapUsers and ldapGroups already exist +// +// old users are removed from the database. a user is considered old if it has not been synced for 1 day +func syncLdapUsers(app *pocketbase.PocketBase, ldapClient *ldap.Conn) SyncResult { + var syncedCount int + var errors []error + + // Create a search request for users + userSearchRequest := ldap.NewSearchRequest( + os.Getenv("LDAP_BASE_DN"), // The base dn to search + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + os.Getenv("LDAP_USER_FILTER"), // The filter to apply + []string{"givenName", "sn", "accountExpires", "uidNumber", "mail", "dn", "cn", "msSFU30NisDomain", "memberOf"}, + nil, + ) + + usersFoundInLdap, err := ldapClient.Search(userSearchRequest) + if err != nil { + return SyncResult{ + Errors: []error{fmt.Errorf("unable to search in ldap for users - %s", err)}, + } + } + + for _, entry := range usersFoundInLdap.Entries { + err := upsertLDAPUser(app, &LDAPUser{ + givenName: entry.GetAttributeValue("givenName"), + sn: entry.GetAttributeValue("sn"), + accountExpires: entry.GetAttributeValue("accountExpires"), + uidNumber: entry.GetAttributeValue("uidNumber"), + mail: entry.GetAttributeValue("mail"), + dn: entry.DN, + cn: entry.GetAttributeValue("cn"), + msSFU30NisDomain: entry.GetAttributeValue("msSFU30NisDomain"), + memberOf: entry.GetAttributeValues("memberOf"), + }) + if err != nil { + errors = append(errors, err) + } else { + syncedCount++ + } + } + + var removedCount int + + // remove old users + // step1: get a timeStamp one day ago + timeStamp := time.Now().AddDate(0, 0, -1) + + // step2: get all users that have not been synced since that timeStamp + records, err := app.Dao().FindRecordsByFilter( + ldapUsersTableName, + "updated < {:timeStamp}", "", 0, 0, + dbx.Params{"timeStamp": timeStamp}, + ) + + if err != nil { + return SyncResult{ + Found: len(usersFoundInLdap.Entries), + Synced: syncedCount, + Errors: []error{fmt.Errorf("unable to get old ldap users from db - %s", err)}, + } + } + + // step3: delete all users that have not been synced since that timeStamp + for _, record := range records { + err := app.Dao().DeleteRecord(record) + if err != nil { + errors = append(errors, fmt.Errorf("unable to remove old user with dn: %s - %s", record.Get("dn"), err)) + } else { + removedCount++ + } + } + + return SyncResult{ + Found: len(usersFoundInLdap.Entries), + Synced: syncedCount, + Removed: removedCount, + Errors: errors, + } +} + +// syncLdap This function syncs the ldap groups and users with the database +func syncLdap(app *pocketbase.PocketBase) { + + // Connect to the LDAP server + conn, err := ldap.DialURL(os.Getenv("LDAP_URL")) + if err != nil { + logger.LogErrorF("unable to connect to ldap server - %s", err) + return + } + defer conn.Close() + + // Bind to the server + err = conn.Bind(os.Getenv("LDAP_BIND_DN"), os.Getenv("LDAP_BIND_PASSWORD")) + if err != nil { + logger.LogErrorF("unable to bind to ldap server - %s", err) + return + } + + // sync groups + groupSyncResult := syncLdapGroups(app, conn) + + // sync users + userSyncResult := syncLdapUsers(app, conn) + + // store sync log in database + collection, err := app.Dao().FindCollectionByNameOrId(ldapSyncLogsTableName) + if err != nil { + logger.LogErrorF("unable to find % table: %s", ldapSyncLogsTableName, err) + return + } + record := models.NewRecord(collection) + form := forms.NewRecordUpsert(app, record) + form.LoadData(map[string]any{ + "usersFound": userSyncResult.Found, + "usersSynced": userSyncResult.Synced, + "usersRemoved": userSyncResult.Removed, + "userSyncErrors": errorsToJSON(userSyncResult.Errors), + "groupsFound": groupSyncResult.Found, + "groupsSynced": groupSyncResult.Synced, + "groupsRemoved": groupSyncResult.Removed, + "groupSyncErrors": errorsToJSON(groupSyncResult.Errors), + }) + if err := form.Submit(); err != nil { + logger.LogErrorF("unable to store ldap sync log: %s", err) + return + } +} + +// errorsToJSON converts an array of errors to a json string +func errorsToJSON(errors []error) string { + var jsonErrors []string + for _, err := range errors { + jsonErrors = append(jsonErrors, err.Error()) + } + jsonString, _ := json.Marshal(jsonErrors) + s := string(jsonString) + return s +} diff --git a/ldapSync/main.go b/ldapSync/main.go new file mode 100644 index 0000000..17b3279 --- /dev/null +++ b/ldapSync/main.go @@ -0,0 +1,102 @@ +/* +Package ldapSync provides a scheduler for syncing ldap users and groups to the database +*/ +package ldapSync + +import ( + "fmt" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/tools/cron" + "gitlab.uni-ulm.de/stuve-it/it-tools/backend/logger" + "os" + "strconv" + "time" +) + +// ldapTimeToUnixTime converts a ldap time string to a time.Time object +func ldapTimeToUnixTime(ldapTimeStampStr string) (time.Time, error) { + + ldapTimeStamp, err := strconv.ParseInt(ldapTimeStampStr, 10, 64) + + if err != nil { + return time.Time{}, fmt.Errorf("error parsing ldap time string: %v", err) + } + + // convert from 100 nanosecond intervals to milliseconds + unixTimeStamp := ldapTimeStamp/1e4 - 1.16444736e13 + + if unixTimeStamp < 0 { + return time.Time{}, fmt.Errorf("error parsing ldap time string: unixTimeStamp is negative") + } + + // Convert milliseconds to seconds + seconds := unixTimeStamp / 1000 + + // Create a time.Time object + t := time.Unix(seconds, 0) + + return t, nil +} + +// InitLdapSync initializes the ldap sync scheduler +// +// the function syncs the ldap users and groups initially and then as defined in the LDAP_SYNC_SCHEDULE env variable (cron syntax) +// +// the function also checks for the existence of the ldapUsers and ldapGroups tables and creates them if they do not exist +func InitLdapSync(app *pocketbase.PocketBase) error { + + // check if ldapGroups table exists + if _, err := app.Dao().FindCollectionByNameOrId(ldapGroupsTableName); err != nil { + // create ldap_groups table if not exists + logger.LogInfoF("creating " + ldapGroupsTableName + " table ...") + if err := createLDAPGroupsTable(app); err != nil { + return err + } + } else { + logger.LogInfoF(ldapGroupsTableName + " table already exists ... skipping creation") + } + + // check if ldapUsers table exists + if _, err := app.Dao().FindCollectionByNameOrId(ldapUsersTableName); err != nil { + // create ldap_users table if not exists + logger.LogInfoF("creating " + ldapUsersTableName + " table ...") + if err := createLDAPUsersTable(app); err != nil { + return err + } + } else { + logger.LogInfoF(ldapUsersTableName + " table already exists ... skipping creation") + } + + // check if ldapSyncLogs table exists + if _, err := app.Dao().FindCollectionByNameOrId(ldapSyncLogsTableName); err != nil { + // create ldapSyncs table if not exists + logger.LogInfoF("creating " + ldapSyncLogsTableName + " table ...") + if err := createLDAPSyncLogsTable(app); err != nil { + return err + } + } else { + logger.LogInfoF(ldapSyncLogsTableName + " table already exists ... skipping creation") + } + + // start sync + scheduler := cron.New() + + // initial sync + logger.LogInfoF("initial LDAP on startup") + syncLdap(app) + logger.LogInfoF("... initial LDAP Sync done") + + ldapSyncSchedule := os.Getenv("LDAP_SYNC_SCHEDULE") + + // syncs ldap every 2 minutes + scheduler.MustAdd("ldapSync", ldapSyncSchedule, func() { + logger.LogInfoF("syncing LDAP ...") + syncLdap(app) + logger.LogInfoF("... LDAP Sync done") + }) + + scheduler.Start() + logger.LogInfoF("ldap sync scheduler started with schedule: %s", ldapSyncSchedule) + + return nil +} diff --git a/ldapSync/models.go b/ldapSync/models.go new file mode 100644 index 0000000..597abfd --- /dev/null +++ b/ldapSync/models.go @@ -0,0 +1,24 @@ +package ldapSync + +type LDAPUser struct { + givenName string + sn string + accountExpires string + uidNumber string + mail string + + dn string + cn string + msSFU30NisDomain string // must be STUVE + memberOf []string +} + +type LDAPGroup struct { + gidNumber string + description string + + dn string + cn string + msSFU30NisDomain string + memberOf []string +} diff --git a/ldapSync/tables.go b/ldapSync/tables.go new file mode 100644 index 0000000..4d6bed5 --- /dev/null +++ b/ldapSync/tables.go @@ -0,0 +1,250 @@ +package ldapSync + +/* + this file contains the code for creating the ldapUsers and ldapGroups tables +*/ + +import ( + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/forms" + "github.com/pocketbase/pocketbase/models" + "github.com/pocketbase/pocketbase/models/schema" + "github.com/pocketbase/pocketbase/tools/types" +) + +const ldapUsersTableName string = "ldap_users" +const ldapGroupsTableName string = "ldap_groups" +const ldapSyncLogsTableName string = "ldap_sync_logs" + +// createLDAPGroupsTable creates ldapGroups table +// +// the function does not check if the ldapGroups table already exists +// returns error +func createLDAPGroupsTable(app *pocketbase.PocketBase) error { + + collection := &models.Collection{} + + form := forms.NewCollectionUpsert(app, collection) + form.Name = ldapGroupsTableName + form.Type = models.CollectionTypeBase + form.ListRule = types.Pointer("@request.auth.id != ''") + form.ViewRule = types.Pointer("@request.auth.id != ''") + form.CreateRule = nil + form.UpdateRule = nil + form.DeleteRule = nil + + // add group ID field + form.Schema.AddField(&schema.SchemaField{ + Name: "gidNumber", + Type: schema.FieldTypeText, + Required: true, + }) + + // add description field + form.Schema.AddField(&schema.SchemaField{ + Name: "description", + Type: schema.FieldTypeText, + Required: false, + }) + + // add common name field + form.Schema.AddField(&schema.SchemaField{ + Name: "cn", + Type: schema.FieldTypeText, + Required: true, + Presentable: true, + }) + + // add distinguished name field + form.Schema.AddField(&schema.SchemaField{ + Name: "dn", + Type: schema.FieldTypeText, + Required: true, + }) + + // create index on cn + form.Indexes = types.JsonArray[string]{ + "CREATE UNIQUE INDEX idx_ldapGroups ON " + ldapGroupsTableName + " (cn, gidNumber, dn)", + } + + // validate and submit (internally it calls app.Dao().SaveCollection(collection) in a transaction) + if err := form.Submit(); err != nil { + return err + } + + // find the ldapGroups collection by name + collection, err := app.Dao().FindCollectionByNameOrId(ldapGroupsTableName) + + if err != nil { + return err + } + + // create form for collection update + form = forms.NewCollectionUpsert(app, collection) + + // add groups field - we cant add this field in the first form because the collection (and the ID) does not exist yet + form.Schema.AddField(&schema.SchemaField{ + Name: "memberOf", + Type: schema.FieldTypeRelation, + Required: false, + Options: &schema.RelationOptions{ + CollectionId: collection.Id, + CascadeDelete: false, + }, + }) + + // validate and submit (internally it calls app.Dao().SaveCollection(collection) in a transaction) + if err := form.Submit(); err != nil { + return err + } + + // return collection id and nil error + return nil +} + +// createLDAPUsersTable creates ldapUsers table +// +// the function does not check if the ldapUsers table already exists +// returns error +func createLDAPUsersTable(app *pocketbase.PocketBase) error { + + // find the ldapGroups collection by name + groupsCollection, err := app.Dao().FindCollectionByNameOrId(ldapGroupsTableName) + if err != nil { + return err + } + + // create ldapUsers table + collection := &models.Collection{} + + // because this is an auth collection, the system will automatically create a username field, a password field, verified field, an email field and an emailVisibility field + + // create form for collection creation + form := forms.NewCollectionUpsert(app, collection) + form.Name = ldapUsersTableName // collection name + form.Type = models.CollectionTypeAuth // collection type set to auth, otherwise login will not work + form.ListRule = types.Pointer("@request.auth.id != ''") // list rule (only authenticated users can list) + form.ViewRule = types.Pointer("@request.auth.id != ''") // view rule (only authenticated users can view) + form.CreateRule = nil // create rule (anyone can create) + form.UpdateRule = nil // update rule (anyone can update) + form.DeleteRule = nil // delete rule (anyone can delete) + + // add common name field, the collection will also have a field named "username" which is the username field. this field is added automatically by the forms.NewCollectionUpsert() function + form.Schema.AddField(&schema.SchemaField{ + Name: "cn", + Type: schema.FieldTypeText, + Required: true, + Presentable: true, + }) + + // add distinguished name field + form.Schema.AddField(&schema.SchemaField{ + Name: "dn", + Type: schema.FieldTypeText, + Required: true, + }) + + // add uidNumber field + form.Schema.AddField(&schema.SchemaField{ + Name: "uidNumber", + Type: schema.FieldTypeText, + Required: true, + }) + + // add surname field + form.Schema.AddField(&schema.SchemaField{ + Name: "sn", + Type: schema.FieldTypeText, + Required: true, + }) + + // add given name field + form.Schema.AddField(&schema.SchemaField{ + Name: "givenName", + Type: schema.FieldTypeText, + Required: true, + }) + + // add account expires field + form.Schema.AddField(&schema.SchemaField{ + Name: "accountExpires", + Type: schema.FieldTypeDate, + Required: false, + }) + + // add groups field + form.Schema.AddField(&schema.SchemaField{ + Name: "memberOf", + Type: schema.FieldTypeRelation, + Required: false, + Options: &schema.RelationOptions{ + CollectionId: groupsCollection.Id, + CascadeDelete: false, + }, + }) + + // create index on username + form.Indexes = types.JsonArray[string]{ + "CREATE UNIQUE INDEX idx_ldapUsers ON " + ldapGroupsTableName + " (cn, uidNumber, dn)", + } + + return form.Submit() +} + +// createLDAPSyncLogsTable creates ldapSyncLogs table +func createLDAPSyncLogsTable(app *pocketbase.PocketBase) error { + // create ldapSyncs table + collection := &models.Collection{} + + // create form for collection creation + form := forms.NewCollectionUpsert(app, collection) + form.Name = ldapSyncLogsTableName // collection name + form.Type = models.CollectionTypeBase // collection type set to auth, otherwise login will not work + + form.Schema.AddField(&schema.SchemaField{ + Name: "usersFound", + Type: schema.FieldTypeNumber, + }) + + form.Schema.AddField(&schema.SchemaField{ + Name: "usersSynced", + Type: schema.FieldTypeNumber, + }) + + form.Schema.AddField(&schema.SchemaField{ + Name: "usersRemoved", + Type: schema.FieldTypeNumber, + }) + + form.Schema.AddField(&schema.SchemaField{ + Name: "userSyncErrors", + Type: schema.FieldTypeJson, + }) + + form.Schema.AddField(&schema.SchemaField{ + Name: "groupsFound", + Type: schema.FieldTypeNumber, + }) + + form.Schema.AddField(&schema.SchemaField{ + Name: "groupsSynced", + Type: schema.FieldTypeNumber, + }) + + form.Schema.AddField(&schema.SchemaField{ + Name: "groupsRemoved", + Type: schema.FieldTypeNumber, + }) + + form.Schema.AddField(&schema.SchemaField{ + Name: "groupSyncErrors", + Type: schema.FieldTypeJson, + }) + + // create index + form.Indexes = types.JsonArray[string]{ + "CREATE UNIQUE INDEX idx_ldapSyncs ON " + ldapSyncLogsTableName + " (created)", + } + + return form.Submit() +} diff --git a/logger/main.go b/logger/main.go new file mode 100644 index 0000000..dedb17e --- /dev/null +++ b/logger/main.go @@ -0,0 +1,47 @@ +/* +Package logger provides a simple logger for the application. +*/ +package logger + +import ( + "fmt" + "github.com/fatih/color" + "log" + "strings" +) + +// LogSuccessF LogSuccess logs a success message to stdout +func LogSuccessF(format string, a ...interface{}) { + logger("success", format, a...) +} + +// LogInfoF LogInfo logs an info message to stdout +func LogInfoF(format string, a ...interface{}) { + logger("info", format, a...) +} + +// LogErrorF LogError logs an error message to stdout +func LogErrorF(format string, a ...interface{}) { + logger("error", format, a...) +} + +// logger logger is a helper function to log messages to stdout +func logger(level string, format string, a ...interface{}) { + var c *color.Color + switch level { + case "success": + c = color.New(color.Bold).Add(color.FgGreen) + case "info": + c = color.New(color.FgBlue) + case "error": + c = color.New(color.FgRed) + } + + date := new(strings.Builder) + log.New(date, "", log.LstdFlags).Print() + + c.Printf("%s - %s\n", + strings.TrimSpace(date.String()), + fmt.Sprintf(format, a...), + ) +} diff --git a/main/main.go b/main/main.go new file mode 100644 index 0000000..9d69687 --- /dev/null +++ b/main/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "github.com/joho/godotenv" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/core" + "gitlab.uni-ulm.de/stuve-it/it-tools/backend/ldapLogin" + "gitlab.uni-ulm.de/stuve-it/it-tools/backend/ldapSync" + "log" +) + +func main() { + + // load env + godotenv.Load(".env.local") + godotenv.Load(".env") + + // create app + app := pocketbase.New() + + // setup ldap sync + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { + return ldapSync.InitLdapSync(app) + }) + + // setup ldap login + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { + return ldapLogin.InitLDAPLogin(app, e) + }) + + // start app + if err := app.Start(); err != nil { + log.Fatal(err) + } +}