Commit 97919393 authored by René Jochum's avatar René Jochum

Initial commit

parents
COMPOSE_PROJECT_NAME=minadmin3
\ No newline at end of file
dist/*
!dist/.gitkeep
examples/
docker-compose.yml
vault-secrets.md
config/*
!config/local.minadmin.io.yaml
src/minadmin_auth_api/
.vscode/tags
[submodule "ui"]
path = ui
url = https://git.webmeisterei.com/minadmin/js-dev.git
[submodule "src/shared"]
path = src/shared
url = https://git.webmeisterei.com/minadmin/shared.git
[submodule "src/minadmin_main"]
path = src/minadmin_main
url = https://git.webmeisterei.com/minadmin/minadmin_main.git
[submodule "src/minadmin_srv_user"]
path = src/minadmin_srv_user
url = https://git.webmeisterei.com/minadmin/minadmin_srv_user.git
This diff is collapsed.
SUBDIRS=src
ifndef CONFIG
CONFIG=${PWD}/config/local.minadmin.io.yaml
endif
DIST_OUT_DIR=${PWD}/dist
DOCKER_OUT_DIR=${DIST_OUT_DIR}/docker
.PHONY: all
all: templates build up logs
.PHONY: templates
templates:
sudo rsync -avP ${PWD}/docker/ ${DOCKER_OUT_DIR}
${PWD}/bin/generate-template.py -c ${CONFIG} docker-compose.yml.sls > ${PWD}/docker-compose.yml
chmod 0600 ${PWD}/docker-compose.yml
${PWD}/bin/generate-template.py -c ${CONFIG} traefik/config/traefik.toml.jinja > ${DOCKER_OUT_DIR}/traefik/config/traefik.toml
${PWD}/bin/generate-template.py -c ${CONFIG} postgres/docker-entrypoint-initdb.d/init-minadmin_main-db.sh.jinja > ${DOCKER_OUT_DIR}/postgres/docker-entrypoint-initdb.d/init-minadmin_main-db.sh
${PWD}/bin/generate-template.py -c ${CONFIG} postgres/docker-entrypoint-initdb.d/init-minadmin_srv_user-db.sh.jinja > ${DOCKER_OUT_DIR}/postgres/docker-entrypoint-initdb.d/init-minadmin_srv_user-db.sh
if [ ! -e ${DOCKER_OUT_DIR}/traefik/config/admins.htpasswd ]; then \
htpasswd -bBc ${DOCKER_OUT_DIR}/traefik/config/admins.htpasswd `${PWD}/bin/get-config-value.py -c ${CONFIG} traefik.email` `${PWD}/bin/get-config-value.py -c ${CONFIG} traefik.password`; \
else \
htpasswd -bB ${DOCKER_OUT_DIR}/traefik/config/admins.htpasswd `${PWD}/bin/get-config-value.py -c ${CONFIG} traefik.email` `${PWD}/bin/get-config-value.py -c ${CONFIG} traefik.password`; \
fi
# SSL
mkdir -p ${DIST_OUT_DIR}/tls
cp ${PWD}/openssl.conf ${DIST_OUT_DIR}/tls/
cp ${PWD}/bin/cert-gen.sh ${DIST_OUT_DIR}/tls/
cd ${DIST_OUT_DIR}/tls; FQDN=`${PWD}/bin/get-config-value.py -c ${CONFIG} minadmin.fqdn` ./cert-gen.sh ${DIST_OUT_DIR}/tls; cd ${PWD}
.PHONY: build
build:
for dir in ${SUBDIRS}; do ${MAKE} -C $$dir $@ || exit 1; done;
docker-compose build
.PHONY: build-go-tools
build-go-tools:
${MAKE} -C go build
.PHONY: up
up:
sudo docker-compose up -d
.PHONY: vault-shell
vault-shell:
docker-compose exec vault sh
.PHONY: down
down:
docker-compose down --remove-orphans
.PHONY: logs
logs:
docker-compose logs -f
.PHONY: logs-traefik
logs-traefik:
docker-compose logs -f traefik
.PHONY: logs-minadmin-api-v1
logs-minadmin-api-v1:
docker-compose logs -f minadmin-api-v1-api minadmin-api-v1-srv
.PHONY: logs-minadmin-auth
logs-minadmin-auth:
docker-compose logs -f minadmin-auth minadmin-auth-srv
.PHONY: logs-dgraph
logs-dgraph:
docker-compose logs -f dgraph-server dgraph-zero dgraph-ratel
.PHONY: restart
restart: down templates up
.PHONY: build-restart
build-restart: templates build restart
.PHONY: rebuild-minadmin-srv-user
rebuild-minadmin-srv-user:
${MAKE} -C src/minadmin_srv_user clean
${MAKE} -C src/minadmin_srv_user build
.PHONY: restart-minadmin-srv-user
restart-minadmin-srv-user: templates
docker-compose up -d --build minadmin-srv-user
.PHONY: rebuild-restart-minadmin-srv-user
rebuild-restart-minadmin-srv-user: rebuild-minadmin-srv-user restart-minadmin-srv-user
.PHONY: rebuild-minadmin-main
rebuild-minadmin-main:
${MAKE} -C src/minadmin_main clean
${MAKE} -C src/minadmin_main build
.PHONY: restart-minadmin-main
restart-minadmin-main: templates
docker-compose up -d --build minadmin-main
.PHONY: rebuild-restart-minadmin-main
rebuild-restart-minadmin-main: rebuild-minadmin-main restart-minadmin-main
.PHONY: vendor
vendor:
for dir in ${SUBDIRS}; do ${MAKE} -C $$dir $@ || exit 1; done;
.PHONY: clean
clean: down
sudo rm -rf ${DIST_OUT_DIR}/*
touch ${DIST_OUT_DIR}/.gitkeep
rm -f ${PWD}/docker-compose.yml
\ No newline at end of file
# minAdmin
## The components of minAdmin
| Component | Description |
|-----------|-------------|
| [minadmin_main](https://git.webmeisterei.com/minadmin/minadmin_main) | The "main" entrypoint for clients - its the Webserver and hosts .well-known/minadmin-configuration. |
| [minadmin_auth](https://git.webmeisterei.com/minadmin/minadmin_auth) | The IDP for Hydra |
| [minadmin_auth_srv_xorm](https://git.webmeisterei.com/minadmin/minadmin_auth_srv_xorm) | Currently the only available backend for "minadmin_auth" above |
| [minadmin_init](https://git.webmeisterei.com/minadmin/minadmin_init) | Creates the "root" user and the minAdmin hydra client |
| [minadmin](https://git.webmeisterei.com/minadmin/minadmin) | Contains the shared golang Package for all other components |
| [agent_zimbra](https://git.webmeisterei.com/minadmin/agent_zimbra) | The first minAdmin agent, for Zimbra |
## External Components used here
| Component | Description |
|-----------|-------------|
| [consul](https://www.consul.io/) | Connects everything together |
| [vault](https://www.vault.io) | Secret Storage - not in use yet |
| [traefik](https://traefik.io/) | The HTTP Router in use |
| [hydra](https://www.ory.sh/docs/guides/master/hydra/) | Hydra is used as OpenID Connect Provider |
| [micro](https://micro.mu/) | Our microframework to connected each piece together |
| [postgresql](https://www.postgresql.org/) | The World's Most Advanced Open Source Relational Database |
## Installation
### Clone this repo
```
git clone --recurse-submodules -j8 https://git.webmeisterei.com/minadmin/dev.git
```
### Setup
- Install go 1.11
I use [gvm](https://github.com/moovweb/gvm) for that.
- Install reqs for the template generator
```
sudo apt install bzr rsync python3-yaml python3-jinja2 apache2-utils
```
- Install docker from the offical [repo](https://docs.docker.com/install/)
- Install docker-compose
```
sudo apt install python-pip
sudo pip install docker-compose
```
- Run it
```
make
```
## Important URLs
- [minAdmin](https://local.minadmin.io)
- [Traefik dashboard](https://local.minadmin.io/traefik)
- [Consul UI](http://consul.local.minadmin.io)
- [Vault UI](http://vault.local.minadmin.io)
## Development URLs
- [micro-web](http://micro.local.minadmin.io)
- [pgAdmin4](http://pgadmin.local.minadmin.io)
### Folder structure
```
src/minadmin Shared between minAdmin and its Agents
src/minadmin/agent Shared Agent base package
src/agent_ispconfig ISPConfig Agent
src/agent_zimbra Zimbra Agent
ui Our Angular Nx Workspace @minadmin/js-dev
Read more about it here: https://nrwl.io/nx/guide-nx-workspace
https://izifortune.com/multiple-packages-repository-with-angular/
ui/apps/minadmin minAdmin main app
ui/libs/oidc OpenID Connect Library for minadmin @minadmin/js-oidc
```
### URLs
```
https://local.minadmin.io/# Login
/status Status of all customer services
/admin Admin overview page
/agents Administrate Agents
/ui Administrate Frontends
/dns Aggregates all agents that provide "dns"
/mail Aggregates all agents that provide "mail"
/php Aggregates all agents that provide "php"
```
```
https://local.minadmin.io/api/
/auth
/health GET Execute health check
/login POST Login func
/logout POST Logout func
/refresh POST Refresh a Token
/user GET List users
POST Create a new User
/me GET Get information about the current logged in user
/<email> GET Get informations about the user with the given "email"
PUT Update user with the given "email"
DELETE Delete user with the given "email"
/api/v1
/health GET Execute health check
/agent GET List Agents
/refresh POST Refresh Agents
/ws GET List workspaces for the current user
POST Create a workspace for the current user
/health GET Execute health check
/<UUID> GET Get informations about the workspace "UUID"
PUT Update the workspace "UUID"
DELETE Delete the workspace "UUID"
/resource GET List resources in the current workspace
/provider GET List providers
/io.minadmin.type.mail.domain GET Get provider Info
POST Create a Resource on that Provider
/<UUID> GET Get the data for the Resource "UUID"
PUT Update Resource "UUID"
DELETE Delete Resource "UUID"
```
### Agent API
gRPC
\ No newline at end of file
#!/bin/bash
# if [ -z $SAN ]; then
# echo "Set SAN with a DNS or IP(e.g. export SAN=IP.1:127.0.0.1,IP.2:172.18.0.2)."
# exit 1
# fi
# if [ ! -e "$1/ca.crt" ]; then
# # Creating basic files/directories
# mkdir -p {certs,crl,newcerts}
# touch index.txt
# echo 1000 > serial
# # CA private key (unencrypted)
# openssl genrsa -out $1/ca.key 4096
# # Certificate Authority (self-signed certificate)
# openssl req -config openssl.conf -new -x509 -days 3650 -sha256 -key $1/ca.key -extensions v3_ca -out $1/ca.crt -subj "/CN=fake-ca"
# fi
# ### Server
# if [ ! -d $1/server ]; then
# mkdir -p $1/server/
# cp $1/ca.crt $1/server/
# # Server private key (unencrypted)
# openssl genrsa -out $1/server/server.key 2048
# # Server certificate signing request (CSR)
# openssl req -config openssl.conf -new -sha256 -key $1/server/server.key -out $1/server.csr -subj "/CN=fake-server"
# # Certificate Authority signs CSR to grant a certificate
# openssl ca -batch -config openssl.conf -extensions server_cert -days 365 -notext -md sha256 -in $1/server.csr -out $1/server/server.crt -cert $1/ca.crt -keyfile $1/ca.key
# fi
# ### Client
# if [ ! -d $1/client ]; then
# mkdir -p $1/client/
# cp $1/ca.crt $1/client/
# # Client private key (unencrypted)
# openssl genrsa -out $1/client/client.key 2048
# # Signed client certificate signing request (CSR)
# openssl req -config openssl.conf -new -sha256 -key $1/client/client.key -out $1/client.csr -subj "/CN=fake-client"
# # Certificate Authority signs CSR to grant a certificate
# openssl ca -batch -config openssl.conf -extensions usr_cert -days 365 -notext -md sha256 -in $1/client.csr -out $1/client/client.crt -cert $1/ca.crt -keyfile $1/ca.key
# fi
## FQDN
if [ ! -d $1/fqdn ]; then
mkdir -p $1/fqdn/
cp $1/ca.crt $1/fqdn/
# FQDN Cert
openssl genrsa -out $1/fqdn/fqdn.key 2048
openssl req -x509 -config openssl.conf -new -sha256 -key $1/fqdn/fqdn.key -out $1/fqdn/fqdn.crt -subj "/CN=${FQDN}"
fi
# Remove CSR's
# rm $1/*.csr
\ No newline at end of file
#!/usr/bin/env python3
from jinja2 import Environment, FileSystemLoader
import yaml
import argparse
import os.path
# https://stackoverflow.com/a/7205107
def merge(a, b, path=None):
"merges b into a"
if path is None: path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
merge(a[key], b[key], path + [str(key)])
elif a[key] == b[key]:
pass # same leaf value
else:
a[key] = b[key]
return a
def main():
curdir = os.path.dirname(__file__)
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--directory", type=str,
help="Templates directory", default=os.path.join(curdir, "..", "templates"))
parser.add_argument("-c", "--config", type=str, help="YAML config file", default=os.path.join(curdir, "..", "config", "example.local.yaml"))
parser.add_argument("template", type=str, help="Template to use")
args = parser.parse_args()
#Load data from YAML into Python dictionary
config = yaml.load(open(args.config))
config_default = config["__default__"]
del config["__default__"]
for k in config.keys():
config[k] = merge(config[k], config_default)
#Load Jinja2 template
env = Environment(loader = FileSystemLoader(args.directory), trim_blocks=True, lstrip_blocks=True)
template = env.get_template(args.template)
#Render the template with data and print the output
print(template.render(config))
if __name__ == "__main__":
main()
#!/usr/bin/env python3
import yaml
import argparse
import os.path
# https://stackoverflow.com/a/7205107
def merge(a, b, path=None):
"merges b into a"
if path is None: path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
merge(a[key], b[key], path + [str(key)])
elif a[key] == b[key]:
pass # same leaf value
else:
a[key] = b[key]
return a
def main():
curdir = os.path.dirname(__file__)
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--config", type=str, help="YAML config file", default=os.path.join(curdir, "..", "config", "example.local.yaml"))
parser.add_argument("key", type=str, help="Key of value to get")
args = parser.parse_args()
#Load data from YAML into Python dictionary
config = yaml.load(open(args.config))
config_default = config["__default__"]
del config["__default__"]
for k in config.keys():
config[k] = merge(config[k], config_default)
dots = args.key.split('.')
result = config
for i in dots:
result = result[i]
print(result)
if __name__ == "__main__":
main()
# Defaults for all services
__default__:
enable: True
domain: local.minadmin.io
host: minadmin
email: admin@minadmin.io
password: EmVJPfggvfT9ojX
enc_password: $$apr1$$hkBicBgH$$yjKWZvkORvywK4qPAYqpx1
expose_ports: False
use_http: False
# Globals used by some minadmin services
minadmin:
enable: True
use_http: True
fqdn: local.minadmin.io
nats_server: nats-server:4222
# OpenID Connect Settings
oidc:
issuer_url: https://auth.webmeisterei.com/auth/realms/minadmin
client_id: "local.minadmin.io"
runner:
enable: True
traefik:
expose_ports: False
host: traefik
ipv4_address: 192.168.48.203
use_http: True
pgadmin:
host: pgadmin
use_http: True
postgresd:
admin_user: admin
admin_pass: kcqQI6iNLQqk9OMO9dpJva6Li5OiSx
main_db: postgres
dbs:
minadmin_main:
user: minadmin_main
password: Jr73tC980msQDd6qxkABVLFIIdS4NE
minadmin_srv_user:
user: minadmin_srv_user
password: gWjj9FTTjsRymokdLFnvLHmRMMVnmg
nats:
expose_ports: True
micro_api:
enable: False
micro_web:
enable: False
host: micro
use_http: True
micro_proxy:
# SECURITY: Theres no authentication for micro_proxy!
enable: False
\ No newline at end of file
{
"datacenter": "localhost",
"data_dir": "/consul/data",
"log_level": "INFO",
"server": true,
"ui": true,
"ports": {
"dns": 53
}
}
\ No newline at end of file
# A simple wrapper for micro to expose ports for consul registrator
FROM microhq/micro:latest
EXPOSE 8082
EXPOSE 8081
EXPOSE 8080
ENTRYPOINT [ "/micro" ]
\ No newline at end of file
FROM debian:buster
RUN apt-get update --allow-releaseinfo-change && apt-get -qy dist-upgrade -o 'DPkg::Options::=--force-confold' -o 'DPkg::Options::=--force-confdef'
RUN apt-get install -qy -o 'DPkg::Options::=--force-confold' -o 'DPkg::Options::=--force-confdef' \
sudo curl vim git-core
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -; \
apt-get install -y gcc g++ make; \
curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -; \
echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list; \
apt-get update && \
apt-get install -y yarn && \
npm install -g npm-check-updates
RUN useradd -ms /bin/bash debian
RUN echo "debian ALL=NOPASSWD: ALL" > /etc/sudoers.d/debian; chmod 600 /etc/sudoers.d/debian
USER debian
WORKDIR /minadmin
CMD ["sleep", "365d"]
\ No newline at end of file
# A simple wrapper for traefik to expose ports for consul registrator
FROM traefik:v1.7.11
EXPOSE 8080
EXPOSE 80
EXPOSE 443
\ No newline at end of file
admin@minadmin.io:$2y$05$gkfbdU3D/G4L0KQ3PQccP.SbW2on01LNx7qZ1BwW5qJDh21piDJMe
################################################################
# Global configuration
################################################################
# Enable debug mode
#
# Optional
# Default: false
#
# debug = true
# Log level
#
# Optional
# Default: "ERROR"
#
# logLevel = "DEBUG"
# Entrypoints to be used by frontends that do not specify any entrypoint.
# Each frontend can specify its own entrypoints.
#
# Optional
# Default: ["http"]
#
defaultEntryPoints = ["http", "https"]
################################################################
# Entrypoints configuration
################################################################
# Entrypoints definition
#
# Optional
# Default:
[entryPoints]
[entryPoints.http]
address = ":80"
[entryPoints.http.forwardedHeaders]
# List of trusted IPs
#
# Required
# Default: []
#
trustedIPs = ["10.0.0.0/8", "192.168.0.0/16"]
[entryPoints.https]
address = ":443"
[entryPoints.https.tls]
minVersion = "VersionTLS12"
cipherSuites = [
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"TLS_RSA_WITH_AES_256_GCM_SHA384"
]
[[entryPoints.https.tls.certificates]]
CertFile = "/etc/traefik/tls/fqdn/fqdn.crt"
KeyFile = "/etc/traefik/tls/fqdn/fqdn.key"
################################################################
# Traefik logs configuration
################################################################
# Traefik logs
# Enabled by default and log to stdout
#
# Optional
#
[traefikLog]
# Sets the filepath for the traefik log. If not specified, stdout will be used.
# Intermediate directories are created if necessary.
#
# Optional
# Default: os.Stdout
#
# filePath = "log/traefik.log"
# Format is either "json" or "common".
#
# Optional
# Default: "common"
#
# format = "common"
################################################################
# Access logs configuration
################################################################
# Enable access logs
# By default it will write to stdout and produce logs in the textual
# Common Log Format (CLF), extended with additional fields.
#
# Optional
#
[accessLog]
# Sets the file path for the access log. If not specified, stdout will be used.
# Intermediate directories are created if necessary.
#
# Optional
# Default: os.Stdout
#
# filePath = "/path/to/log/log.txt"
# Format is either "json" or "common".
#
# Optional
# Default: "common"
#
# format = "common"
################################################################
# API and dashboard configuration
################################################################
# Enable API and dashboard
[api]
# Name of the related entry point
#
# Optional
# Default: "traefik"
#
# entryPoint = "traefik"
# Enabled Dashboard
#
# Optional
# Default: true
#
dashboard = true
# Enable debug mode.
# This will install HTTP handlers to expose Go expvars under /debug/vars and
# pprof profiling data under /debug/pprof/.
# Additionally, the log level will be set to DEBUG.
#
# Optional
# Default: false
#
debug = true
################################################################
# Ping configuration
################################################################
# Enable ping
[ping]
# Name of the related entry point
#
# Optional
# Default: "traefik"
#
# entryPoint = "traefik"
################################################################
# Docker configuration backend
################################################################
# Enable Docker configuration backend
# [docker]
# Docker server endpoint. Can be a tcp or a unix socket endpoint.
#
# Required
# Default: "unix:///var/run/docker.sock"
#
# endpoint = "tcp://10.10.10.10:2375"
# Default domain used.
# Can be overridden by setting the "traefik.domain" label on a container.