ompluscator/dynamic-struct

Golang package for editing struct's fields during runtime and mapping structs to other structs.

Golang dynamic struct

Package dynamic struct provides possibility to dynamically, in runtime, extend or merge existing defined structs or to provide completely new struct.

Main features:

• Building completely new struct in runtime
• Extending existing struct in runtime
• Merging multiple structs in runtime
• Adding new fields into struct
• Removing existing fields from struct
• Modifying fields' types and tags
• Easy reading of dynamic structs
• Mapping dynamic struct with set values to existing struct
• Make slices and maps of dynamic structs

Works out-of-the-box with:

• https://github.com/go-playground/form
• https://github.com/go-playground/validator
• https://github.com/leebenson/conform
• https://golang.org/pkg/encoding/json/
• ...

Benchmarks

Environment:

• MacBook Pro (13-inch, Early 2015), 2,7 GHz Intel Core i5
• go version go1.11 darwin/amd64

goos: darwin
goarch: amd64
pkg: github.com/ompluscator/dynamic-struct
BenchmarkClassicWay_NewInstance-4                 2000000000     0.34 ns/op
BenchmarkNewStruct_NewInstance-4                    10000000      141 ns/op
BenchmarkNewStruct_NewInstance_Parallel-4           20000000     89.6 ns/op
BenchmarkExtendStruct_NewInstance-4                 10000000      135 ns/op
BenchmarkExtendStruct_NewInstance_Parallel-4        20000000     89.5 ns/op
BenchmarkMergeStructs_NewInstance-4                 10000000      140 ns/op
BenchmarkMergeStructs_NewInstance_Parallel-4        20000000     94.3 ns/op

Add new struct

package main

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/ompluscator/dynamic-struct"
)

func main() {
	instance := dynamicstruct.NewStruct().
		AddField("Integer", 0, `json:"int"`).
		AddField("Text", "", `json:"someText"`).
		AddField("Float", 0.0, `json:"double"`).
		AddField("Boolean", false, "").
		AddField("Slice", []int{}, "").
		AddField("Anonymous", "", `json:"-"`).
		Build().
		New()

	data := []byte(`
{
    "int": 123,
    "someText": "example",
    "double": 123.45,
    "Boolean": true,
    "Slice": [1, 2, 3],
    "Anonymous": "avoid to read"
}
`)

	err := json.Unmarshal(data, &instance)
	if err != nil {
		log.Fatal(err)
	}

	data, err = json.Marshal(instance)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(string(data))
	// Out:
	// {"int":123,"someText":"example","double":123.45,"Boolean":true,"Slice":[1,2,3]}
}

Extend existing struct

package main

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/ompluscator/dynamic-struct"
)

type Data struct {
	Integer int `json:"int"`
}

func main() {
	instance := dynamicstruct.ExtendStruct(Data{}).
		AddField("Text", "", `json:"someText"`).
		AddField("Float", 0.0, `json:"double"`).
		AddField("Boolean", false, "").
		AddField("Slice", []int{}, "").
		AddField("Anonymous", "", `json:"-"`).
		Build().
		New()

	data := []byte(`
{
    "int": 123,
    "someText": "example",
    "double": 123.45,
    "Boolean": true,
    "Slice": [1, 2, 3],
    "Anonymous": "avoid to read"
}
`)

	err := json.Unmarshal(data, &instance)
	if err != nil {
		log.Fatal(err)
	}

	data, err = json.Marshal(instance)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(string(data))
	// Out:
	// {"int":123,"someText":"example","double":123.45,"Boolean":true,"Slice":[1,2,3]}
}

Merge existing structs

package main

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/ompluscator/dynamic-struct"
)

type DataOne struct {
	Integer int     `json:"int"`
	Text    string  `json:"someText"`
	Float   float64 `json:"double"`
}

type DataTwo struct {
	Boolean bool
	Slice []int
	Anonymous string `json:"-"`
}

func main() {
	instance := dynamicstruct.MergeStructs(DataOne{}, DataTwo{}).
		Build().
		New()

	data := []byte(`
{
"int": 123,
"someText": "example",
"double": 123.45,
"Boolean": true,
"Slice": [1, 2, 3],
"Anonymous": "avoid to read"
}
`)

	err := json.Unmarshal(data, &instance)
	if err != nil {
		log.Fatal(err)
	}

	data, err = json.Marshal(instance)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(string(data))
	// Out:
	// {"int":123,"someText":"example","double":123.45,"Boolean":true,"Slice":[1,2,3]}
}

Read dynamic struct

package main

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/ompluscator/dynamic-struct"
)

type DataOne struct {
	Integer int     `json:"int"`
	Text    string  `json:"someText"`
	Float   float64 `json:"double"`
}

type DataTwo struct {
	Boolean bool
	Slice []int
	Anonymous string `json:"-"`
}

func main() {
	instance := dynamicstruct.MergeStructs(DataOne{}, DataTwo{}).
		Build().
		New()

	data := []byte(`
{
"int": 123,
"someText": "example",
"double": 123.45,
"Boolean": true,
"Slice": [1, 2, 3],
"Anonymous": "avoid to read"
}
`)

	err := json.Unmarshal(data, &instance)
	if err != nil {
		log.Fatal(err)
	}

	reader := dynamicstruct.NewReader(instance)

	fmt.Println("Integer", reader.GetField("Integer").Int())
	fmt.Println("Text", reader.GetField("Text").String())
	fmt.Println("Float", reader.GetField("Float").Float64())
	fmt.Println("Boolean", reader.GetField("Boolean").Bool())
	fmt.Println("Slice", reader.GetField("Slice").Interface().([]int))
	fmt.Println("Anonymous", reader.GetField("Anonymous").String())

	var dataOne DataOne
	err = reader.ToStruct(&dataOne)
	fmt.Println(err, dataOne)

	var dataTwo DataTwo
	err = reader.ToStruct(&dataTwo)
	fmt.Println(err, dataTwo)
	// Out:
	// Integer 123
	// Text example
	// Float 123.45
	// Boolean true
	// Slice [1 2 3]
	// Anonymous
	// <nil> {123 example 123.45}
	// <nil> {true [1 2 3] }
}

Make a slice of dynamic struct

package main

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/ompluscator/dynamic-struct"
)

type Data struct {
	Integer   int     `json:"int"`
	Text      string  `json:"someText"`
	Float     float64 `json:"double"`
	Boolean   bool
	Slice     []int
	Anonymous string `json:"-"`
}

func main() {
	definition := dynamicstruct.ExtendStruct(Data{}).Build()

	slice := definition.NewSliceOfStructs()

	data := []byte(`
[
	{
		"int": 123,
		"someText": "example",
		"double": 123.45,
		"Boolean": true,
		"Slice": [1, 2, 3],
		"Anonymous": "avoid to read"
	}
]
`)

	err := json.Unmarshal(data, &slice)
	if err != nil {
		log.Fatal(err)
	}

	data, err = json.Marshal(slice)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(string(data))
	// Out:
	// [{"Boolean":true,"Slice":[1,2,3],"int":123,"someText":"example","double":123.45}]

	reader := dynamicstruct.NewReader(slice)
	readersSlice := reader.ToSliceOfReaders()
	for k, v := range readersSlice {
		var value Data
		err := v.ToStruct(&value)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Println(k, value)
	}
	// Out:
	// 0 {123 example 123.45 true [1 2 3] }
}

Make a map of dynamic struct

package main

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/ompluscator/dynamic-struct"
)

type Data struct {
	Integer   int     `json:"int"`
	Text      string  `json:"someText"`
	Float     float64 `json:"double"`
	Boolean   bool
	Slice     []int
	Anonymous string `json:"-"`
}

func main() {
	definition := dynamicstruct.ExtendStruct(Data{}).Build()

	mapWithStringKey := definition.NewMapOfStructs("")

	data := []byte(`
{
	"element": {
		"int": 123,
		"someText": "example",
		"double": 123.45,
		"Boolean": true,
		"Slice": [1, 2, 3],
		"Anonymous": "avoid to read"
	}
}
`)

	err := json.Unmarshal(data, &mapWithStringKey)
	if err != nil {
		log.Fatal(err)
	}

	data, err = json.Marshal(mapWithStringKey)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(string(data))
	// Out:
	// {"element":{"int":123,"someText":"example","double":123.45,"Boolean":true,"Slice":[1,2,3]}}

	reader := dynamicstruct.NewReader(mapWithStringKey)
	readersMap := reader.ToMapReaderOfReaders()
	for k, v := range readersMap {
		var value Data
		err := v.ToStruct(&value)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Println(k, value)
	}
	// Out:
	// element {123 example 123.45 true [1 2 3] }
}

dynamic-struct's Issues

Need Capacity as a input to NewSliceOfStructs()

NewSliceOfStructs() currently returns slice of dynamic struct with 0 capcity. This is causing memory leak issues in my case (where we need to unmarshall sqlx.Rows output to slice of dynamic struct). It would be better if it takes capacity as input and returns the slice with the given capacity.

Replace current Struct?

There is anyway to replace struct? its really important for me now

Can we generate struct into struct?

for example how we create similar struct from example:

type Yaml struct {
	Name     string     `yaml:"name"`
	Version  string     `yaml:"version"`
	Datasets []Datasets `yaml:"datasets"`
}

type Datasets struct {
	Name   string   `yaml:"name"`
	Fields []Fields `yaml:"fields"`
}

type Fields struct {
	Tag  string `yaml:"tag"`
	Name string `yaml:"name"`
	Type string `yaml:"type"`
}

Not an issue per se

Nice piece of software you have here!

I've got a doubt about if is possible to have methods on the dynamic structs built with this library. Or in other words, if the dynamic structs can implement a interface

Some light about it will be awesome

Thank you for this package!

This package has been instrumental in some of the most critical aspects of an unannounced project and I just wanted to thank you for putting together such a well though-out and implemented package. It's a joy to use, thank you! <3

Support embedded dynamic struct

Thanks for making this great tool. It is very useful for GraphQL query which requires dynamic struct sometimes.

One feature seems missing is not supported embedded dynamic struct. For example,

func main() {
  childStruct := dynamicstruct.NewStruct().
    AddField("Child", 0, `json:"int"`).
    Build()

  childInstance := childStruct.New()
  content, err := json.Marshal(childInstance)
  if err != nil {
    log.Fatal(err)
  }
  fmt.Println(string(content))

  parentStruct := dynamicstruct.NewStruct().
    AddField("Parent", &childInstance, "").Build()

  parentInstance := parentStruct.New()
  content, err = json.Marshal(parentInstance)
  if err != nil {
    log.Fatal(err)
  }

  fmt.Println(string(content))
}

Result will be:

{"int":0}
{"Parent":null}

Seems like this is due to embedded dynamic struct instance is not initialized, so if explicitly init like below, it will marshal correctly

func main() {
        childStruct := dynamicstruct.NewStruct().
                AddField("Child", 0, `json:"int"`).
                Build()

        childInstance := childStruct.New()
        content, err := json.Marshal(childInstance)
        if err != nil {
                log.Fatal(err)
        }
        fmt.Println(string(content))

        parentStruct := dynamicstruct.NewStruct().
                AddField("Parent", &childInstance, "").Build()

        parentInstance := parentStruct.New()

                data := []byte(`
                {
                        "Parent": {
                          "Child": 0
                        }
                }
                `)

                err = json.Unmarshal(data, &parentInstance)
                if err != nil {
                        log.Fatal(err)
                }

        content, err = json.Marshal(parentInstance)
        if err != nil {
                log.Fatal(err)
        }

        fmt.Println(string(content))
}

Marshal result is

{"int":0}
{"Parent":{"Child":0}}

Error: unreadable: could not resolve interface type

Hi, I'm getting this error with example's code on README.

instance := dynamicstruct.NewStruct().
AddField("Integer", 0, json:"int").
AddField("Text", "", json:"someText").
AddField("Float", 0.0, json:"double").
AddField("Boolean", false, "").
AddField("Slice", []int{}, "").
AddField("Anonymous", "", json:"-").
Build().
New()

Builder to allow add field and build in different statement

I tried below and the dynamic struct at the end is empty.

instance := dynamicstruct.NewStruct() //empty dynamic struct
for _, attr := range Attributes {
instance.AddField(strings.Title(attr.Name), "", `json:"`+attr.MappedTo+`"`) //add fields
}
instance.Build().New() //build in the end

Can we assign a structure variable to a structure field

instance := dynamicstruct.NewStruct(). AddField("Integer", 0, json:"int"). AddField("Text", "", json:"someText"). AddField("Float", 0.0, json:"double"). AddField("Boolean", false, ""). AddField("Slice", []interface{}{}, ""). AddField("Anonymous", "", json:"-"`).
Build().
New()

data := []byte(`

{
"int": 123,
"someText": "example",
"double": 123.45,
"Boolean": true,
"Slice": {
"Test": 124,
"Test2": 234
},
"Anonymous": "avoid to read"
}
)
The error message is as follows：
json: cannot unmarshal object into Go struct field .Slice of type []interface {}
exit status 1

slice of map

[solved]

thanks

Getting partial struct

I'm trying to get this work, but getting unexpected results.

package main

import (
	"encoding/json"
	"fmt"
	"log"

	dynamicstruct "github.com/Ompluscator/dynamic-struct"
)

func main() {
	i := dynamicstruct.NewStruct().
		AddField("AlarmName", "", `json:"alarm_name"`).
		AddField("ComparisonOperator", "", `json:"comparison_operator"`).
		AddField("EvaluationPeriods", 0, `json:"evaluation_periods"`).
		AddField("Threshold", 0, `json:"threshold"`).

		// Optional
		AddField("MetricName", "", `json:"metric_name,omitempty"`).
		AddField("Namespace", "", `json:"namespace,omitempty"`).
		AddField("Period", 0, `json:"period,omitempty"`).
		AddField("Statistic", "", `json:"statistic,omitempty"`).
		AddField("TreatMissingData", "", `json:"treat_missing_data,omitempty"`).
		Build().New()

	data := []byte(`
{
	"ActionsEnabled": null,
	"AlarmActions": ["${aws_autoscaling_policy.jenkins_cluster_scale_up_policy.id}"],
	"AlarmDescription":"CPU utilization above 5% during the last minute",
	"AlarmName":"JenkinsClusterScaleUpAlarm",
	"ComparisonOperator":"GreaterThanOrEqualToThreshold",
	"DatapointsToAlarm":null,
	"Dimensions":[{"Name":"ClusterName","Value":"jenkins-cluster"}],
	"EvaluateLowSampleCountPercentile":null,
	"EvaluationPeriods":1,
	"ExtendedStatistic":null,
	"InsufficientDataActions":null,
	"MetricName":"CPUUtilization",
	"Metrics":null,
	"Namespace":"AWS/ECS",
	"OKActions":null,
	"Period":null,
	"Statistic":"Average",
	"Threshold":20,
	"TreatMissingData":"notBreaching",
	"Unit":null
}
	`)
	var err error
	err = json.Unmarshal(data, &i)
	if err != nil {
		log.Fatal(err)
	}

	data, err = json.Marshal(i)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(string(data))
}

Actual output:

{"namespace":"AWS/ECS","statistic":"Average","alarm_name":"","comparison_operator":"","evaluation_periods":0,"threshold":20}

Expected output:

{"namespace":"AWS/ECS","statistic":"Average","alarm_name":"JenkinsClusterScaleUpAlarm","comparison_operator":"GreaterThanOrEqualToThreshold","evaluation_periods": 1,"threshold":20,"treat_missing_data":"notBreaching","metric_name":"CPUUtilization"}

Any advice is highly appreciated 😅!

Struct fields are in random order

Hi,
We were creating dynamic structs and using them with parquet-go to write parquet files, which works fine.
However, the order of the columns in our parquet files is different each time, we dug a little bit; and it turns out that in your struct builder you are storing fields in a map, which doesn't guarantee order when you create the final definition.
Would it be ok to support that? I can write up a Pull Request for that quickly as I've done it already.
Thanks

How to set the value to the Structure after creating the new dynamic struct

Live Example
https://play.golang.org/p/xEhdtxy-zSU

package main

import (
	"fmt"
	"reflect"
	dynamicstruct "github.com/ompluscator/dynamic-struct"
)



type User struct {
	UserEmail string
	UserPass  string
	Test      string
}

type DataTwo struct {
	UserAge string
}


func main() {
	instance := dynamicstruct.MergeStructs(User{}, DataTwo{}).
		Build().
		New()
	fmt.Println(reflect.TypeOf(instance))
	val := reflect.ValueOf(instance).Elem()
	fmt.Println(val.FieldByName("UserEmail").Interface().(string))

}

I want to set the values to the struct something like that so how to achieve that ?

a := instance {
  UserEmail: "[email protected]"
}

can support sqlx？

Can I cast to a dynamic struct?

I am creating a dynamic struct and storing the definition ([]byte{} data) to a database.

I have an interface{} which I need to cast back to the dynamic type, once I load it out of the database.
Is this possible?

use with sqlx and mysql

I tried like this, but it doesn't return the rows

definition := dynamicstruct.NewStruct().
AddField("A", 0, ```db:"a" json:"a"```).
AddField("R", 0, ```db:"r" json:"r"```).
AddField("D", "", ```db:"d" json:"d"```).
AddField("G", 0, ```db:"g" json:"g"```).
Build()
a := definition.NewSliceOfStructs()

db.Select(&a, "SELECT a, r, d, g FROM articoli;")

while with the real struct it works

type dati struct {
A uint64 ```db:"a" json:"a"``` R uint64 db:"r" json:"r"
D string ```db:"d" json:"d"``` G uint64 db:"g" json:"g"
}
var a []dati
db.Select(&a, "SELECT a, r, d, g FROM articoli;")
``
Am I doing something wrong?

I can use to "encoding/xml"?

Is posssible use "encoding/xml"?

Custom structure of array type

type Top struct {
	Id string `json:"id"`
	User      []User `json:"user"`
}

type User struct {
	FirstName string `json:"first_name"`
	LastName   string    `json:"last_name"`
}

how to set the field user as type []User？

Loading the struct from a buffer

Is it possible to load the struct from a buffer containing the struct fields at the proper locations? I tried to make an array with the dynamic struct as the type but it threw an error.
code looks like this

dynStruct := dynamicstruct.NewStruct().AddField(...).Build().New()
buf := make([]dynStruct, len)

Since this didn't work I'm wondering if its possible to overlay the struct over a buffer that I create

failing to unmashall yaml with dynamic struct

hi there,
I need to unmashall dynamically some type I don't know in advance in yaml.
I hope I could use dynamic-struct but it is failing.
here is my code

import (
	"fmt"
	"testing"

	"github.com/davecgh/go-spew/spew"
	dynamicstruct "github.com/ompluscator/dynamic-struct"
	"gopkg.in/yaml.v3"
)

func TestReadFromYaml(t *testing.T) {
	var data = `
content-type: application/vnd.talend.iam-account
name: foo
data:
  key: value
`
	type DataResource struct {
		Key string `yaml:"key"`
	}

	type TopResource struct {
		Name        string `yaml:"name"`
		ContentType string `yaml:"content-type"`
		//		Data        DataResource `yaml:"data"`
	}

	dataResource := dynamicstruct.ExtendStruct(TopResource{}).
		AddField("Data", DataResource{}, `yaml:"data"`).
		Build().New()

	//	dataResource := TopResource{}
	if err := yaml.Unmarshal([]byte(data), &dataResource); err != nil {
		fmt.Println(err.Error())
	}
	spew.Dump(dataResource)
	reader := dynamicstruct.NewReader(dataResource)
	fmt.Println("Name", reader.GetField("Name").String())
}

and here is the output

(map[interface {}]interface {}) (len=3) {
 (string) (len=4) "name": (string) (len=3) "foo",
 (string) (len=4) "data": (map[string]interface {}) (len=1) {
  (string) (len=3) "key": (string) (len=5) "value"
 },
 (string) (len=12) "content-type": (string) (len=34) "application/vnd.talend.iam-account"
}
Failed to continue - bad access

First the data field is not of type DataResource which is what I expected
and then the reader fails with a bad access error.
Any idea ?

Non deterministic

It appears there is an element of randomness.

dataSchema := dynamicstruct.NewStruct()
for _, aSeries := range df.Series {
dataSchema.AddField(...)
}

schemaStruct := dataSchema.Build()

spew.Dump(schemaStruct)

Results

TRY 1:

(*dynamicstruct.dynamicStructImpl)(0xc0001f0fc0)({
 definition: (*reflect.rtype)(0xc0000a6540)(struct { E *string "parquet:\"name=e, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\""; A *string "parquet:\"name=a, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\""; B *string "parquet:\"name=b, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\""; C *string "parquet:\"name=c, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\""; D *string "parquet:\"name=d, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\"" })
})

TRY 2:

(*dynamicstruct.dynamicStructImpl)(0xc0000129f0)({
 definition: (*reflect.rtype)(0xc0001123c0)(struct { A *string "parquet:\"name=a, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\""; B *string "parquet:\"name=b, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\""; C *string "parquet:\"name=c, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\""; D *string "parquet:\"name=d, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\""; E *string "parquet:\"name=e, type=UTF8, encoding=PLAIN_DICTIONARY, repetitiontype=OPTIONAL\"" })

provide types(string,int etc) directly in place of instance of types

Hi, I just came across this library and started playing with it for one of my use case. I have some dynamic domain types in my application which can be created through UI and user can provide name, types for all attributes etc. which is then saved to DB.
Now for the data I need to create dynamic Struct from the DB which has type already stored.
So is it possible to add field like below

dynamicstruct.NewStruct()
	.AddField("Name", string, `json:"name"`).Build().New()

Can't add more than 1 field with same type

instance := dynamicstruct.NewStruct().
AddField("Integer", 10, json:"int").
AddField("Text", "", json:"someText").
AddField("Text", "", json:"someText1").
AddField("Float", 0.0, json:"double").
AddField("Boolean", false, "").
AddField("Slice", []int{}, "").
AddField("Anonymous", "", json:"-").
Build().
New()

This will not work, will added only "someText1" field, but not someText"

super small, but I found a wrong test error message

TestReaderImpl_GetAllFields
mentions the number 10, but there are 13 in the struct ;-)

Interesting library!

convert interface to struct

Hey! This may sound naive, I am new to Go.
I was wondering if there is a way to convert interface returned by New() while building a dynamic struct to a struct type.

Expose anonymous/embedded struct member.

I see that last year support for embedded structs was added, but it doesn't enable the builder to create a type with embedded structs.

Something like this:

	type stitle struct{ Title string }
	type stext struct{ Text string }

	dstruct := dynamicstruct.NewStruct().
		AddField("", stitle{}, ``).
		AddField("", stext{}, ``).
		Build().
		New()

	type compare struct {
		stitle
		stext 
	}

	if dstruct != compare{} {
		fmt.Println("expected equal")
	}

Besides the general idea of creating a new struct, do you think this is a reasonable interface? In particular if name == "", then set anonymous: true.

How to change a type of a field

I'm looking for the example how to change the type of a field