diff --git a/pkg/template/example/project.json b/pkg/template/example/project.json new file mode 100644 index 000000000000..105c44b4b1cf --- /dev/null +++ b/pkg/template/example/project.json @@ -0,0 +1,157 @@ +{ + "id": "example1", + "name": "my-awesome-php-app", + "description": "Example PHP application with PostgreSQL database", + "buildConfig": [ + { + "name": "mfojtik/nginx-php-app", + "type": "docker", + "sourceUri": "https://raw.githubusercontent.com/mfojtik/phpapp/master/Dockerfile", + "imageRepository": "int.registry.com:5000/mfojtik/phpapp" + }, + { + "name": "postgres", + "type": "docker", + "imageRepository": "registry.hub.docker.com/postgres", + "sourceUri": "https://raw.githubusercontent.com/docker-library/postgres/docker/9.2/Dockerfile" + } + ], + "imageRepository": [ + { + "name": "mfojtik/nginx-php-app", + "url": "internal.registry.com:5000/mfojtik/phpapp" + }, + { + "name": "postgres", + "url": "registry.hub.docker.com/postgres" + } + ], + "parameters": [ + { + "name": "DB_PASSWORD", + "description": "PostgreSQL admin user password", + "type": "string", + "generate": "[a-zA-Z0-9]{8}" + }, + { + "name": "DB_USER", + "description": "PostgreSQL username", + "type": "string", + "generate": "admin[a-zA-Z0-9]{4}" + }, + { + "name": "DB_NAME", + "description": "PostgreSQL database name", + "type": "string", + "generate": "[GET:http://broken.url/test]" + }, + { + "name": "SAMPLE_VAR", + "description": "Sample", + "type": "string", + "value": "foo" + } + ], + "serviceLinks": [ + { + "from": "database", + "export": [ + { + "name": "POSTGRES_ADMIN_USERNAME", + "value": "${DB_USER}" + }, + { + "name": "POSTGRES_ADMIN_PASSWORD", + "value": "${DB_PASSWORD}" + }, + { + "name": "POSTGRES_DATABASE_NAME", + "value": "${DB_NAME}" + } + ], + "to": "frontend" + } + ], + "services": [ + { + "name": "database", + "description": "Standalone PostgreSQL 9.2 database service", + "labels": { + "name": "database-service" + }, + "deploymentConfig": { + "deployment": { + "podTemplate": { + "containers": [ + { + "name": "postgresql-1", + "image": { + "name": "postgres", + "tag": "9.2" + }, + "env": [ + { + "name": "POSTGRES_ADMIN_USERNAME", + "value": "${DB_USER}" + }, + { + "name": "POSTGRES_ADMIN_PASSWORD", + "value": "${DB_PASSWORD}" + }, + { + "name": "POSTGRES_DATABASE_NAME", + "value": "${DB_NAME}" + }, + { + "name": "FOO", + "value": "${BAR}" + } + ], + "ports": [ + { + "containerPort": 5432, + "hostPort": 5432 + } + ] + } + ] + } + } + } + }, + { + "name": "frontend", + "description": "Sample PHP 5.2 application served by NGINX", + "labels": { + "name": "frontend-service" + }, + "deploymentConfig": { + "deployment": { + "podTemplate": { + "containers": [ + { + "name": "nginx-php-app", + "hooks": { + "prestart": { + "cmd": "import_database.sh" + }, + "url": "git://github.com/user/myapp-hooks.git" + }, + "image": { + "name": "mfojtik/nginx-php-app", + "tag": "latest" + }, + "ports": [ + { + "containerPort": 8080, + "hostPort": 8080 + } + ] + } + ] + } + } + } + } + ] +} diff --git a/pkg/template/generator.go b/pkg/template/generator.go new file mode 100644 index 000000000000..aaa558afa435 --- /dev/null +++ b/pkg/template/generator.go @@ -0,0 +1,121 @@ +package template + +import ( + "fmt" + "math/rand" + "regexp" + "strings" + + "github.com/openshift/origin/pkg/template/generator" +) + +var valueExp = regexp.MustCompile(`(\$\{([a-zA-Z0-9\_]+)\})`) + +type ParamHash map[string]Parameter + +// Generate the value for the Parameter if the default Value is not set and the +// Generator field is specified. Otherwise, just return the default Value +func (p *Parameter) GenerateValue() error { + if p.Value != "" || p.Generate == "" { + return nil + } + + g := generator.Generator{} + generatedValue, err := g.Generate(p.Generate).Value() + + if err != nil { + return err + } + p.Value = generatedValue + + return nil +} + +// The string representation of PValue +// +func (s PValue) String() string { + return string(s) +} + +// Replace references to parameters in PValue with their values. +// The format is specified in the `valueExp` constant ${PARAM_NAME}. +// +// If the referenced parameter is not defined, then the substitution is ignored. +func (s *PValue) Substitute(params ParamHash) { + newValue := *s + + for _, match := range valueExp.FindAllStringSubmatch(string(newValue), -1) { + // If the Parameter is not defined, then leave the value as it is + if params[match[2]].Value == "" { + continue + } + newValue = PValue(strings.Replace(string(newValue), match[1], params[match[2]].Value, 1)) + } + + *s = newValue +} + +// Generate Value field for defined Parameters. +// If the Parameter define Generate, then the Value is generated based +// on that template. The template is a pseudo-regexp formatted string. +// +// Example: +// +// s := generate.Template("[a-zA-Z0-9]{4}") +// // s: "Ga0b" +// +// s := generate.Template("[GET:http://example.com/new]") +// // s: +func (p *Template) ProcessParameters() { + rand.Seed(p.RandomSeed) + + for i, _ := range p.Parameters { + if err := p.Parameters[i].GenerateValue(); err != nil { + fmt.Printf("ERROR: Unable to process parameter %s: %v\n", p.Parameters[i].Name, err) + p.Parameters[i].Value = p.Parameters[i].Generate + } + } +} + +// A shorthand method to get list of *all* container defined in the Template +// template +func (p *Template) Containers() []*Container { + var result []*Container + for _, s := range p.Services { + result = append(result, s.Containers()...) + } + return result +} + +// Convert Parameter slice to more effective data structure +func (p *Template) ParameterHash() ParamHash { + paramHash := make(ParamHash) + for _, p := range p.Parameters { + paramHash[p.Name] = p + } + return paramHash +} + +// Process all Env variables in the Project template and replace parameters +// referenced in their values with the Parameter values. +// +// The replacement is done in Containers and ServiceLinks. +func (p *Template) SubstituteEnvValues() { + + params := p.ParameterHash() + + for _, container := range p.Containers() { + (*container).Env.Process(params) + } + + for s, _ := range p.ServiceLinks { + p.ServiceLinks[s].Export.Process(params) + } +} + +// Substitute referenced parameters in Env values with parameter values. +func (e *Env) Process(params ParamHash) { + for i, _ := range *e { + (*e)[i].Value.Substitute(params) + } +} diff --git a/pkg/template/generator/from_url.go b/pkg/template/generator/from_url.go new file mode 100644 index 000000000000..d7ee68c1f185 --- /dev/null +++ b/pkg/template/generator/from_url.go @@ -0,0 +1,26 @@ +package generator + +import ( + "io/ioutil" + "net/http" + "strings" +) + +func httpGet(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + return string(body), err +} + +func replaceUrlWithData(s *string, expresion string) error { + result, err := httpGet(expresion[5 : len(expresion)-1]) + if err != nil { + return err + } + *s = strings.Replace(*s, expresion, strings.TrimSpace(result), 1) + return nil +} diff --git a/pkg/template/generator/generator.go b/pkg/template/generator/generator.go new file mode 100644 index 000000000000..9b259e331ae6 --- /dev/null +++ b/pkg/template/generator/generator.go @@ -0,0 +1,34 @@ +package generator + +import "fmt" + +type GeneratorType interface { + Value() (string, error) +} + +type ExpresionGenerator struct { + expr string +} + +func (g ExpresionGenerator) Value() (string, error) { + return FromTemplate(g.expr) +} + +type PasswordGenerator struct { + length int +} + +func (g PasswordGenerator) Value() (string, error) { + return FromTemplate(fmt.Sprintf("[\\a]{%d}", g.length)) +} + +type Generator struct{} + +func (g Generator) Generate(t string) GeneratorType { + switch t { + case "password": + return PasswordGenerator{length: 8} + default: + return ExpresionGenerator{expr: t} + } +} diff --git a/pkg/template/generator/template.go b/pkg/template/generator/template.go new file mode 100644 index 000000000000..64776002ea9c --- /dev/null +++ b/pkg/template/generator/template.go @@ -0,0 +1,122 @@ +package generator + +import ( + "fmt" + "math/rand" + "regexp" + "strconv" + "strings" +) + +const ( + Alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + Numerals = "0123456789" + Ascii = Alphabet + Numerals + "~!@#$%^&*()-_+={}[]\\|<,>.?/\"';:`" +) + +var ( + rangeExp = regexp.MustCompile(`([\\]?[a-zA-Z0-9]\-?[a-zA-Z0-9]?)`) + generatorsExp = regexp.MustCompile(`\[([a-zA-Z0-9\-\\]+)\](\{([0-9]+)\})`) + remoteExp = regexp.MustCompile(`\[GET\:(http(s)?:\/\/(.+))\]`) +) + +type GeneratorExprRanges [][]byte + +func randomInt(n int) int { + return rand.Intn(n) +} + +func alphabetSlice(from, to byte) (string, error) { + leftPos := strings.Index(Ascii, string(from)) + rightPos := strings.LastIndex(Ascii, string(to)) + if leftPos > rightPos { + return "", fmt.Errorf("Invalid range specified: %s-%s", string(from), string(to)) + } + return Ascii[leftPos:rightPos], nil +} + +func replaceWithGenerated(s *string, expresion string, ranges [][]byte, length int) error { + var alphabet string + for _, r := range ranges { + switch string(r[0]) + string(r[1]) { + case `\w`: + alphabet += Ascii + case `\d`: + alphabet += Numerals + case `\a`: + alphabet += Alphabet + Numerals + default: + if slice, err := alphabetSlice(r[0], r[1]); err != nil { + return err + } else { + alphabet += slice + } + } + } + if len(alphabet) == 0 { + return fmt.Errorf("Empty range in expresion: %s", expresion) + } + result := make([]byte, length, length) + for i := 0; i <= length-1; i++ { + result[i] = alphabet[randomInt(len(alphabet))] + } + *s = strings.Replace(*s, expresion, string(result), 1) + return nil +} + +func findExpresionPos(s string) GeneratorExprRanges { + matches := rangeExp.FindAllStringIndex(s, -1) + result := make(GeneratorExprRanges, len(matches), len(matches)) + for i, r := range matches { + result[i] = []byte{s[r[0]], s[r[1]-1]} + } + return result +} + +func rangesAndLength(s string) (string, int, error) { + l := strings.LastIndex(s, "{") + // If the length ({}) is not specified in expresion, + // then assume the length is 1 character + // + if l > 0 { + expr := s[0:strings.LastIndex(s, "{")] + length, err := parseLength(s) + return expr, length, err + } else { + return s, 1, nil + } +} + +func parseLength(s string) (int, error) { + lengthStr := string(s[strings.LastIndex(s, "{")+1 : len(s)-1]) + if l, err := strconv.Atoi(lengthStr); err != nil { + return 0, fmt.Errorf("Unable to parse length from %v", s) + } else { + return l, nil + } +} + +func FromTemplate(template string) (string, error) { + result := template + genMatches := generatorsExp.FindAllStringIndex(template, -1) + remMatches := remoteExp.FindAllStringIndex(template, -1) + // Parse [a-z]{} types + for _, r := range genMatches { + ranges, length, err := rangesAndLength(template[r[0]:r[1]]) + if err != nil { + return "", err + } + positions := findExpresionPos(ranges) + if err := replaceWithGenerated(&result, template[r[0]:r[1]], positions, length); err != nil { + return "", err + } + } + // Parse [GET:] type + // + for _, r := range remMatches { + if err := replaceUrlWithData(&result, template[r[0]:r[1]]); err != nil { + return "", err + } + } + return result, nil +} diff --git a/pkg/template/generator_test.go b/pkg/template/generator_test.go new file mode 100644 index 000000000000..342f41158e2f --- /dev/null +++ b/pkg/template/generator_test.go @@ -0,0 +1,60 @@ +package template + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "strings" + "testing" + "time" +) + +const projectExampleJSON = "./example/project.json" + +var projectTempl Template + +func TestTemplateUnmarshal(t *testing.T) { + jsonFile, _ := ioutil.ReadFile(projectExampleJSON) + err := json.Unmarshal(jsonFile, &projectTempl) + if err != nil { + t.Errorf("Unable to parse the sample project.json: %v", err) + } + projectTempl.RandomSeed = time.Now().UnixNano() +} + +func TestProcessParameters(t *testing.T) { + projectTempl.ProcessParameters() + + for _, p := range projectTempl.Parameters { + if p.Value == "" { + t.Errorf("Failed to process '%s' parameter", p.Name) + } + fmt.Printf("%s -> %s = %s\n", p.Name, p.Generate, p.Value) + } +} + +func TestSubstituteEnvValues(t *testing.T) { + projectTempl.SubstituteEnvValues() + + for _, c := range projectTempl.Containers() { + for _, e := range c.Env { + if strings.Contains(string(e.Value), "${") { + if e.Name != "FOO" { + t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) + } + } + fmt.Printf("%s=%s\n", e.Name, e.Value) + } + } + + for _, s := range projectTempl.ServiceLinks { + for _, e := range s.Export { + if strings.Contains(string(e.Value), "${") { + if e.Name != "FOO" { + t.Errorf("Failed to substitute %s environment variable: %s", e.Name, e.Value) + } + } + fmt.Printf("%s=%s\n", e.Name, e.Value) + } + } +} diff --git a/pkg/template/service_links.go b/pkg/template/service_links.go new file mode 100644 index 000000000000..ea5a6f6f7f13 --- /dev/null +++ b/pkg/template/service_links.go @@ -0,0 +1,72 @@ +package template + +import "fmt" + +func (e *Env) Append(env *Env) { + *e = append(*e, *env...) +} + +func (e *Env) Exists(name string) bool { + for _, env := range *e { + if env.Name == name { + return true + } + } + return false +} + +func (s *Service) Containers() []*Container { + result := make([]*Container, len((*s).DeploymentConfig.Deployment.PodTemplate.Containers)) + + for i, _ := range s.DeploymentConfig.Deployment.PodTemplate.Containers { + result[i] = &s.DeploymentConfig.Deployment.PodTemplate.Containers[i] + } + + return result +} + +func (s *Service) ContainersEnv() []*Env { + var result []*Env + for _, c := range s.Containers() { + result = append(result, &c.Env) + } + return result +} + +func (s *Service) AddEnv(env Env) { + fmt.Printf("s.Containers() %+v\n", s.Containers()) + for _, c := range s.Containers() { + (*c).Env = append(c.Env, env...) + } +} + +func (p *Template) ServiceByName(name string) *Service { + for i, _ := range p.Services { + if p.Services[i].Name == name { + return &p.Services[i] + } + } + return nil +} + +func (p *Template) ProcessServiceLinks() { + var ( + fromService, toService *Service + ) + + for i, _ := range p.ServiceLinks { + fromService = p.ServiceByName(p.ServiceLinks[i].From) + if fromService == nil { + fmt.Printf("ERROR: Invalid FROM service in links: %+v\n", p.ServiceLinks[i].From) + continue + } + + toService = p.ServiceByName(p.ServiceLinks[i].To) + if toService == nil { + fmt.Printf("ERROR: Invalid TO service in links: %+v\n", p.ServiceLinks[i].To) + continue + } + + toService.AddEnv(p.ServiceLinks[i].Export) + } +} diff --git a/pkg/template/service_links_test.go b/pkg/template/service_links_test.go new file mode 100644 index 000000000000..679ac9ee4292 --- /dev/null +++ b/pkg/template/service_links_test.go @@ -0,0 +1,17 @@ +package template + +import "testing" + +func TestServiceLinks(t *testing.T) { + projectTempl.ProcessParameters() + projectTempl.ProcessServiceLinks() + + s := projectTempl.ServiceByName("frontend") + for _, env := range s.ContainersEnv() { + for _, export := range projectTempl.ServiceLinks[0].Export { + if env.Exists(export.Name) == false { + t.Errorf("Failed to export %s variable via serviceLinks to %s", export.Name, s.Name) + } + } + } +} diff --git a/pkg/template/types.go b/pkg/template/types.go new file mode 100644 index 000000000000..3b38dac525ee --- /dev/null +++ b/pkg/template/types.go @@ -0,0 +1,87 @@ +package template + +import "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + +type ( + Uri string + PValue string +) + +type Template struct { + api.JSONBase `json:",inline" yaml:",inline"` + BuildConfig []BuildConfig `json:"buildConfig" yaml:"buildConfig"` + ImageRepository []ImageRepository `json:"imageRepository" yaml:"imageRepository"` + Parameters []Parameter `json:"parameters" yaml:"parameters"` + ServiceLinks []ServiceLink `json:"serviceLinks" yaml:"serviceLinks"` + Services []Service `json:"services" yaml:"services"` + + RandomSeed int64 +} + +type ImageRepository struct { + Name string `json:"name" yaml:"name"` + Url Uri `json:"url" yaml:"url"` +} + +type BuildConfig struct { + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + SourceUri Uri `json:"sourceUri" yaml:"sourceUri"` + ImageRepository string `json:"imageRepository" yaml:"imageRepository"` +} + +type Parameter struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` + Type string `json:"type" yaml:"type"` + Generate string `json:"generate" yaml:"generate"` + Value string `json:"value" yaml:"value"` +} + +type Env []struct { + Name string `json:"name" yaml:"name"` + Value PValue `json:"value" yaml:"value"` +} + +type ServiceLink struct { + From string `json:"from" yaml:"from"` + To string `json:"to" yaml:"to"` + Export Env `json:"export" yaml:"export"` +} + +type DeploymentConfig struct { + Deployment Deployment `json:"deployment" yaml:"deployment"` +} + +type Deployment struct { + PodTemplate PodTemplate `json:"podTemplate" yaml:"podTemplate"` +} + +type PodTemplate struct { + Containers []Container `json:"containers" yaml:"containers"` + Replicas int `json:"replicas" yaml:"replicas"` +} + +type Image struct { + Name string `json:"name" yaml:"name"` + Tag string `json:"tag" yaml:"tag"` +} + +type ContainerPort struct { + ContainerPort int `json:"containerPort" yaml:"containerPort"` + HostPort int `json:"hostPort" yaml:"hostPort"` +} + +type Container struct { + Name string `json:"name" yaml:"name"` + Image Image `json:"image" yaml:"image"` + Env Env `json:"env" yaml:"env"` + Ports []ContainerPort `json:"ports" yaml:"ports"` +} + +type Service struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` + Labels map[string]PValue `json:"labels" yaml:"labels"` + DeploymentConfig DeploymentConfig `json:"deploymentConfig" yaml:"deploymentConfig"` +}