From 5507383c809156ca7218f6140a328cde0aae2af1 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Wed, 5 Apr 2017 09:44:04 -0700 Subject: [PATCH] Swarm to support node-local networks - regardless of the network driver datascope Signed-off-by: Alessandro Boch --- api/specs.pb.go | 274 ++++++++++-------- api/specs.proto | 5 + .../networkallocator/drivers_network_linux.go | 19 ++ ..._network.go => drivers_network_windows.go} | 2 - .../networkallocator/networkallocator.go | 147 +++++++--- manager/controlapi/common.go | 12 +- 6 files changed, 299 insertions(+), 160 deletions(-) create mode 100644 manager/allocator/networkallocator/drivers_network_linux.go rename manager/allocator/networkallocator/{drivers_network.go => drivers_network_windows.go} (91%) diff --git a/api/specs.pb.go b/api/specs.pb.go index 7bf3593c66..c9ea260466 100644 --- a/api/specs.pb.go +++ b/api/specs.pb.go @@ -642,6 +642,10 @@ type NetworkSpec struct { // swarm internally created only and it was identified by the name // "ingress" and the label "com.docker.swarm.internal": "true". Ingress bool `protobuf:"varint,7,opt,name=ingress,proto3" json:"ingress,omitempty"` + // ConfigFrom indicates that the network specific configuration + // for this network will be provided via another network, locally + // on the node where this network is being plumbed. + ConfigFrom string `protobuf:"bytes,8,opt,name=config_from,json=configFrom,proto3" json:"config_from,omitempty"` } func (m *NetworkSpec) Reset() { *m = NetworkSpec{} } @@ -2006,6 +2010,12 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) { } i++ } + if len(m.ConfigFrom) > 0 { + dAtA[i] = 0x42 + i++ + i = encodeVarintSpecs(dAtA, i, uint64(len(m.ConfigFrom))) + i += copy(dAtA[i:], m.ConfigFrom) + } return i, nil } @@ -2534,6 +2544,10 @@ func (m *NetworkSpec) Size() (n int) { if m.Ingress { n += 2 } + l = len(m.ConfigFrom) + if l > 0 { + n += 1 + l + sovSpecs(uint64(l)) + } return n } @@ -2817,6 +2831,7 @@ func (this *NetworkSpec) String() string { `IPAM:` + strings.Replace(fmt.Sprintf("%v", this.IPAM), "IPAMOptions", "IPAMOptions", 1) + `,`, `Attachable:` + fmt.Sprintf("%v", this.Attachable) + `,`, `Ingress:` + fmt.Sprintf("%v", this.Ingress) + `,`, + `ConfigFrom:` + fmt.Sprintf("%v", this.ConfigFrom) + `,`, `}`, }, "") return s @@ -5242,6 +5257,35 @@ func (m *NetworkSpec) Unmarshal(dAtA []byte) error { } } m.Ingress = bool(v != 0) + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigFrom", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpecs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthSpecs + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ConfigFrom = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSpecs(dAtA[iNdEx:]) @@ -5883,119 +5927,121 @@ var ( func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) } var fileDescriptorSpecs = []byte{ - // 1824 bytes of a gzipped FileDescriptorProto + // 1844 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x57, 0x4f, 0x73, 0x1b, 0x49, - 0x15, 0xb7, 0x6c, 0x59, 0x7f, 0xde, 0xc8, 0x89, 0xd2, 0x24, 0x61, 0xac, 0xb0, 0xb2, 0xa2, 0x0d, - 0xc1, 0xcb, 0x16, 0x72, 0x61, 0xa8, 0x25, 0x4b, 0x58, 0x40, 0xb2, 0x84, 0x63, 0x8c, 0x1d, 0x55, - 0xdb, 0x1b, 0xc8, 0x49, 0xd5, 0x9e, 0x69, 0x4b, 0x53, 0x1e, 0x75, 0x0f, 0xdd, 0x3d, 0xda, 0xd2, - 0x8d, 0xe3, 0x56, 0xae, 0x9c, 0x5d, 0x1c, 0xf8, 0x32, 0xb9, 0x41, 0x71, 0xe2, 0xe4, 0x62, 0xfd, - 0x15, 0xf8, 0x00, 0x50, 0xdd, 0xd3, 0xa3, 0x3f, 0xc9, 0x78, 0x93, 0x2a, 0xc2, 0xad, 0xfb, 0xcd, - 0xef, 0xf7, 0xfa, 0xf5, 0xeb, 0x5f, 0xf7, 0x7b, 0x03, 0x8e, 0x8c, 0xa8, 0x27, 0x5b, 0x91, 0xe0, - 0x8a, 0x23, 0xe4, 0x73, 0xef, 0x82, 0x8a, 0x96, 0xfc, 0x8a, 0x88, 0xf1, 0x45, 0xa0, 0x5a, 0x93, - 0x1f, 0xd7, 0x1c, 0x35, 0x8d, 0xa8, 0x05, 0xd4, 0xee, 0x0e, 0xf9, 0x90, 0x9b, 0xe1, 0x8e, 0x1e, - 0x59, 0x6b, 0x7d, 0xc8, 0xf9, 0x30, 0xa4, 0x3b, 0x66, 0x76, 0x16, 0x9f, 0xef, 0xf8, 0xb1, 0x20, - 0x2a, 0xe0, 0xcc, 0x7e, 0xdf, 0x7c, 0xf3, 0x3b, 0x61, 0xd3, 0xe4, 0x53, 0xf3, 0x32, 0x0f, 0xa5, - 0x63, 0xee, 0xd3, 0x93, 0x88, 0x7a, 0x68, 0x1f, 0x1c, 0xc2, 0x18, 0x57, 0x86, 0x2b, 0xdd, 0x5c, - 0x23, 0xb7, 0xed, 0xec, 0x6e, 0xb5, 0xde, 0x0e, 0xaa, 0xd5, 0x9e, 0xc3, 0x3a, 0xf9, 0xd7, 0x57, - 0x5b, 0x2b, 0x78, 0x91, 0x89, 0x7e, 0x05, 0x15, 0x9f, 0xca, 0x40, 0x50, 0x7f, 0x20, 0x78, 0x48, - 0xdd, 0xd5, 0x46, 0x6e, 0xfb, 0xd6, 0xee, 0xf7, 0xb2, 0x3c, 0xe9, 0xc5, 0x31, 0x0f, 0x29, 0x76, - 0x2c, 0x43, 0x4f, 0xd0, 0x3e, 0xc0, 0x98, 0x8e, 0xcf, 0xa8, 0x90, 0xa3, 0x20, 0x72, 0xd7, 0x0c, - 0xfd, 0x07, 0x37, 0xd1, 0x75, 0xec, 0xad, 0xa3, 0x19, 0x1c, 0x2f, 0x50, 0xd1, 0x11, 0x54, 0xc8, - 0x84, 0x04, 0x21, 0x39, 0x0b, 0xc2, 0x40, 0x4d, 0xdd, 0xbc, 0x71, 0xf5, 0xc9, 0xb7, 0xba, 0x6a, - 0x2f, 0x10, 0xf0, 0x12, 0xbd, 0xe9, 0x03, 0xcc, 0x17, 0x42, 0x8f, 0xa1, 0xd8, 0xef, 0x1d, 0x77, - 0x0f, 0x8e, 0xf7, 0xab, 0x2b, 0xb5, 0xcd, 0x57, 0x97, 0x8d, 0x7b, 0xda, 0xc7, 0x1c, 0xd0, 0xa7, - 0xcc, 0x0f, 0xd8, 0x10, 0x6d, 0x43, 0xa9, 0xbd, 0xb7, 0xd7, 0xeb, 0x9f, 0xf6, 0xba, 0xd5, 0x5c, - 0xad, 0xf6, 0xea, 0xb2, 0x71, 0x7f, 0x19, 0xd8, 0xf6, 0x3c, 0x1a, 0x29, 0xea, 0xd7, 0xf2, 0x5f, - 0xff, 0xb5, 0xbe, 0xd2, 0xfc, 0x3a, 0x07, 0x95, 0xc5, 0x20, 0xd0, 0x63, 0x28, 0xb4, 0xf7, 0x4e, - 0x0f, 0x5e, 0xf4, 0xaa, 0x2b, 0x73, 0xfa, 0x22, 0xa2, 0xed, 0xa9, 0x60, 0x42, 0xd1, 0x23, 0x58, - 0xef, 0xb7, 0xbf, 0x3c, 0xe9, 0x55, 0x73, 0xf3, 0x70, 0x16, 0x61, 0x7d, 0x12, 0x4b, 0x83, 0xea, - 0xe2, 0xf6, 0xc1, 0x71, 0x75, 0x35, 0x1b, 0xd5, 0x15, 0x24, 0x60, 0x36, 0x94, 0xbf, 0xe4, 0xc1, - 0x39, 0xa1, 0x62, 0x12, 0x78, 0x1f, 0x58, 0x22, 0x9f, 0x41, 0x5e, 0x11, 0x79, 0x61, 0xa4, 0xe1, - 0x64, 0x4b, 0xe3, 0x94, 0xc8, 0x0b, 0xbd, 0xa8, 0xa5, 0x1b, 0xbc, 0x56, 0x86, 0xa0, 0x51, 0x18, - 0x78, 0x44, 0x51, 0xdf, 0x28, 0xc3, 0xd9, 0xfd, 0x7e, 0x16, 0x1b, 0xcf, 0x50, 0x36, 0xfe, 0x67, - 0x2b, 0x78, 0x81, 0x8a, 0x9e, 0x42, 0x61, 0x18, 0xf2, 0x33, 0x12, 0x1a, 0x4d, 0x38, 0xbb, 0x0f, - 0xb3, 0x9c, 0xec, 0x1b, 0xc4, 0xdc, 0x81, 0xa5, 0xa0, 0x27, 0x50, 0x88, 0x23, 0x9f, 0x28, 0xea, - 0x16, 0x0c, 0xb9, 0x91, 0x45, 0xfe, 0xd2, 0x20, 0xf6, 0x38, 0x3b, 0x0f, 0x86, 0xd8, 0xe2, 0xd1, - 0x21, 0x94, 0x18, 0x55, 0x5f, 0x71, 0x71, 0x21, 0xdd, 0x62, 0x63, 0x6d, 0xdb, 0xd9, 0xfd, 0x34, - 0x53, 0x8c, 0x09, 0xa6, 0xad, 0x14, 0xf1, 0x46, 0x63, 0xca, 0x54, 0xe2, 0xa6, 0xb3, 0xea, 0xe6, - 0xf0, 0xcc, 0x01, 0xfa, 0x05, 0x94, 0x28, 0xf3, 0x23, 0x1e, 0x30, 0xe5, 0x96, 0x6e, 0x0e, 0xa4, - 0x67, 0x31, 0x3a, 0x99, 0x78, 0xc6, 0xd0, 0x6c, 0xc1, 0xc3, 0xf0, 0x8c, 0x78, 0x17, 0x6e, 0xf9, - 0x3d, 0xb7, 0x31, 0x63, 0x74, 0x0a, 0x90, 0x1f, 0x73, 0x9f, 0x36, 0x77, 0xe0, 0xce, 0x5b, 0xa9, - 0x46, 0x35, 0x28, 0xd9, 0x54, 0x27, 0x1a, 0xc9, 0xe3, 0xd9, 0xbc, 0x79, 0x1b, 0x36, 0x96, 0xd2, - 0xda, 0xfc, 0x47, 0x1e, 0x4a, 0xe9, 0x59, 0xa3, 0x36, 0x94, 0x3d, 0xce, 0x14, 0x09, 0x18, 0x15, - 0x56, 0x5e, 0x99, 0x27, 0xb3, 0x97, 0x82, 0x34, 0xeb, 0xd9, 0x0a, 0x9e, 0xb3, 0xd0, 0x6f, 0xa0, - 0x2c, 0xa8, 0xe4, 0xb1, 0xf0, 0xa8, 0xb4, 0xfa, 0xda, 0xce, 0x56, 0x48, 0x02, 0xc2, 0xf4, 0x8f, - 0x71, 0x20, 0xa8, 0xce, 0xb2, 0xc4, 0x73, 0x2a, 0x7a, 0x0a, 0x45, 0x41, 0xa5, 0x22, 0x42, 0x7d, - 0x9b, 0x44, 0x70, 0x02, 0xe9, 0xf3, 0x30, 0xf0, 0xa6, 0x38, 0x65, 0xa0, 0xa7, 0x50, 0x8e, 0x42, - 0xe2, 0x19, 0xaf, 0xee, 0xba, 0xa1, 0x7f, 0x94, 0x45, 0xef, 0xa7, 0x20, 0x3c, 0xc7, 0xa3, 0xcf, - 0x01, 0x42, 0x3e, 0x1c, 0xf8, 0x22, 0x98, 0x50, 0x61, 0x25, 0x56, 0xcb, 0x62, 0x77, 0x0d, 0x02, - 0x97, 0x43, 0x3e, 0x4c, 0x86, 0x68, 0xff, 0x7f, 0xd2, 0xd7, 0x82, 0xb6, 0x0e, 0x01, 0xc8, 0xec, - 0xab, 0x55, 0xd7, 0x27, 0xef, 0xe5, 0xca, 0x9e, 0xc8, 0x02, 0x1d, 0x3d, 0x84, 0xca, 0x39, 0x17, - 0x1e, 0x1d, 0xd8, 0x5b, 0x53, 0x36, 0x9a, 0x70, 0x8c, 0x2d, 0xd1, 0x17, 0xea, 0x40, 0x71, 0x48, - 0x19, 0x15, 0x81, 0xe7, 0x82, 0x59, 0xec, 0x71, 0xe6, 0x85, 0x4c, 0x20, 0x38, 0x66, 0x2a, 0x18, - 0x53, 0xbb, 0x52, 0x4a, 0xec, 0x94, 0xa1, 0x28, 0x92, 0x2f, 0xcd, 0x3f, 0x00, 0x7a, 0x1b, 0x8b, - 0x10, 0xe4, 0x2f, 0x02, 0xe6, 0x1b, 0x61, 0x95, 0xb1, 0x19, 0xa3, 0x16, 0x14, 0x23, 0x32, 0x0d, - 0x39, 0xf1, 0xad, 0x58, 0xee, 0xb6, 0x92, 0x7a, 0xd9, 0x4a, 0xeb, 0x65, 0xab, 0xcd, 0xa6, 0x38, - 0x05, 0x35, 0x0f, 0xe1, 0x5e, 0xe6, 0x96, 0xd1, 0x2e, 0x54, 0x66, 0x22, 0x1c, 0x04, 0x76, 0x91, - 0xce, 0xed, 0xeb, 0xab, 0x2d, 0x67, 0xa6, 0xd6, 0x83, 0x2e, 0x76, 0x66, 0xa0, 0x03, 0xbf, 0xf9, - 0xe7, 0x32, 0x6c, 0x2c, 0x49, 0x19, 0xdd, 0x85, 0xf5, 0x60, 0x4c, 0x86, 0xd4, 0xc6, 0x98, 0x4c, - 0x50, 0x0f, 0x0a, 0x21, 0x39, 0xa3, 0xa1, 0x16, 0xb4, 0x3e, 0xd4, 0x1f, 0xbd, 0xf3, 0x4e, 0xb4, - 0x7e, 0x67, 0xf0, 0x3d, 0xa6, 0xc4, 0x14, 0x5b, 0x32, 0x72, 0xa1, 0xe8, 0xf1, 0xf1, 0x98, 0x30, - 0xfd, 0x74, 0xae, 0x6d, 0x97, 0x71, 0x3a, 0xd5, 0x99, 0x21, 0x62, 0x28, 0xdd, 0xbc, 0x31, 0x9b, - 0x31, 0xaa, 0xc2, 0x1a, 0x65, 0x13, 0x77, 0xdd, 0x98, 0xf4, 0x50, 0x5b, 0xfc, 0x20, 0x51, 0x64, - 0x19, 0xeb, 0xa1, 0xe6, 0xc5, 0x92, 0x0a, 0xb7, 0x98, 0x64, 0x54, 0x8f, 0xd1, 0xcf, 0xa0, 0x30, - 0xe6, 0x31, 0x53, 0xd2, 0x2d, 0x99, 0x60, 0x37, 0xb3, 0x82, 0x3d, 0xd2, 0x08, 0xfb, 0xb4, 0x5b, - 0x38, 0xea, 0xc1, 0x1d, 0xa9, 0x78, 0x34, 0x18, 0x0a, 0xe2, 0xd1, 0x41, 0x44, 0x45, 0xc0, 0x7d, - 0xfb, 0x34, 0x6d, 0xbe, 0x75, 0x28, 0x5d, 0xdb, 0xe4, 0xe0, 0xdb, 0x9a, 0xb3, 0xaf, 0x29, 0x7d, - 0xc3, 0x40, 0x7d, 0xa8, 0x44, 0x71, 0x18, 0x0e, 0x78, 0x94, 0x54, 0xa9, 0x44, 0x4f, 0xef, 0x91, - 0xb2, 0x7e, 0x1c, 0x86, 0xcf, 0x13, 0x12, 0x76, 0xa2, 0xf9, 0x04, 0xdd, 0x87, 0xc2, 0x50, 0xf0, - 0x38, 0x92, 0xae, 0x63, 0x92, 0x61, 0x67, 0xe8, 0x0b, 0x28, 0x4a, 0xea, 0x09, 0xaa, 0xa4, 0x5b, - 0x31, 0x5b, 0xfd, 0x38, 0x6b, 0x91, 0x13, 0x03, 0xc1, 0xf4, 0x9c, 0x0a, 0xca, 0x3c, 0x8a, 0x53, - 0x0e, 0xda, 0x84, 0x35, 0xa5, 0xa6, 0xee, 0x46, 0x23, 0xb7, 0x5d, 0xea, 0x14, 0xaf, 0xaf, 0xb6, - 0xd6, 0x4e, 0x4f, 0x5f, 0x62, 0x6d, 0xd3, 0x2f, 0xe8, 0x88, 0x4b, 0xc5, 0xc8, 0x98, 0xba, 0xb7, - 0x4c, 0x6e, 0x67, 0x73, 0xf4, 0x12, 0xc0, 0x67, 0x72, 0xe0, 0x99, 0x2b, 0xeb, 0xde, 0x36, 0xbb, - 0xfb, 0xf4, 0xdd, 0xbb, 0xeb, 0x1e, 0x9f, 0xd8, 0x2a, 0xb2, 0x71, 0x7d, 0xb5, 0x55, 0x9e, 0x4d, - 0x71, 0xd9, 0x67, 0x32, 0x19, 0xa2, 0x0e, 0x38, 0x23, 0x4a, 0x42, 0x35, 0xf2, 0x46, 0xd4, 0xbb, - 0x70, 0xab, 0x37, 0x97, 0x85, 0x67, 0x06, 0x66, 0x3d, 0x2c, 0x92, 0xb4, 0x82, 0x75, 0xa8, 0xd2, - 0xbd, 0x63, 0x72, 0x95, 0x4c, 0xd0, 0x47, 0x00, 0x3c, 0xa2, 0x6c, 0x20, 0x95, 0x1f, 0x30, 0x17, - 0xe9, 0x2d, 0xe3, 0xb2, 0xb6, 0x9c, 0x68, 0x03, 0x7a, 0xa0, 0x1f, 0x6d, 0xe2, 0x0f, 0x38, 0x0b, - 0xa7, 0xee, 0x77, 0xcc, 0xd7, 0x92, 0x36, 0x3c, 0x67, 0xe1, 0x14, 0x6d, 0x81, 0x63, 0x74, 0x21, - 0x83, 0x21, 0x23, 0xa1, 0x7b, 0xd7, 0xe4, 0x03, 0xb4, 0xe9, 0xc4, 0x58, 0xf4, 0x39, 0x24, 0xd9, - 0x90, 0xee, 0xbd, 0x9b, 0xcf, 0xc1, 0x06, 0x3b, 0x3f, 0x07, 0xcb, 0x41, 0xbf, 0x04, 0x88, 0x44, - 0x30, 0x09, 0x42, 0x3a, 0xa4, 0xd2, 0xbd, 0x6f, 0x36, 0x5d, 0xcf, 0x7c, 0xad, 0x67, 0x28, 0xbc, - 0xc0, 0xa8, 0x7d, 0x0e, 0xce, 0xc2, 0x6d, 0xd3, 0xb7, 0xe4, 0x82, 0x4e, 0xed, 0x05, 0xd6, 0x43, - 0x9d, 0x92, 0x09, 0x09, 0xe3, 0xa4, 0x13, 0x2e, 0xe3, 0x64, 0xf2, 0xf3, 0xd5, 0x27, 0xb9, 0xda, - 0x2e, 0x38, 0x0b, 0xaa, 0x43, 0x1f, 0xc3, 0x86, 0xa0, 0xc3, 0x40, 0x2a, 0x31, 0x1d, 0x90, 0x58, - 0x8d, 0xdc, 0x5f, 0x1b, 0x42, 0x25, 0x35, 0xb6, 0x63, 0x35, 0xaa, 0x0d, 0x60, 0x7e, 0x78, 0xa8, - 0x01, 0x8e, 0x16, 0x85, 0xa4, 0x62, 0x42, 0x85, 0xae, 0xb6, 0x3a, 0xe7, 0x8b, 0x26, 0x2d, 0x5e, - 0x49, 0x89, 0xf0, 0x46, 0xe6, 0xed, 0x28, 0x63, 0x3b, 0xd3, 0x8f, 0x41, 0x7a, 0x43, 0xec, 0x63, - 0x60, 0xa7, 0xcd, 0x7f, 0xe7, 0xa0, 0xb2, 0xd8, 0x34, 0xa0, 0xbd, 0xa4, 0xd8, 0x9b, 0x2d, 0xdd, - 0xda, 0xdd, 0x79, 0x57, 0x93, 0x61, 0x4a, 0x6b, 0x18, 0x6b, 0x67, 0x47, 0xba, 0xbf, 0x37, 0x64, - 0xf4, 0x53, 0x58, 0x8f, 0xb8, 0x50, 0xe9, 0x13, 0x96, 0x9d, 0x60, 0x2e, 0xd2, 0x52, 0x94, 0x80, - 0x9b, 0x23, 0xb8, 0xb5, 0xec, 0x0d, 0x3d, 0x82, 0xb5, 0x17, 0x07, 0xfd, 0xea, 0x4a, 0xed, 0xc1, - 0xab, 0xcb, 0xc6, 0x77, 0x97, 0x3f, 0xbe, 0x08, 0x84, 0x8a, 0x49, 0x78, 0xd0, 0x47, 0x3f, 0x84, - 0xf5, 0xee, 0xf1, 0x09, 0xc6, 0xd5, 0x5c, 0x6d, 0xeb, 0xd5, 0x65, 0xe3, 0xc1, 0x32, 0x4e, 0x7f, - 0xe2, 0x31, 0xf3, 0x31, 0x3f, 0x9b, 0xf5, 0xba, 0x7f, 0x5b, 0x05, 0xc7, 0xbe, 0xec, 0x1f, 0xfa, - 0x77, 0x68, 0x23, 0x29, 0xe5, 0xe9, 0x95, 0x5d, 0x7d, 0x67, 0x45, 0xaf, 0x24, 0x04, 0x7b, 0xc6, - 0x0f, 0xa1, 0x12, 0x44, 0x93, 0xcf, 0x06, 0x94, 0x91, 0xb3, 0xd0, 0xb6, 0xbd, 0x25, 0xec, 0x68, - 0x5b, 0x2f, 0x31, 0xe9, 0xf7, 0x22, 0x60, 0x8a, 0x0a, 0x66, 0x1b, 0xda, 0x12, 0x9e, 0xcd, 0xd1, - 0x17, 0x90, 0x0f, 0x22, 0x32, 0xb6, 0x6d, 0x48, 0xe6, 0x0e, 0x0e, 0xfa, 0xed, 0x23, 0xab, 0xc1, - 0x4e, 0xe9, 0xfa, 0x6a, 0x2b, 0xaf, 0x0d, 0xd8, 0xd0, 0x50, 0x3d, 0xed, 0x04, 0xf4, 0x4a, 0xe6, - 0xed, 0x2f, 0xe1, 0x05, 0x8b, 0xd6, 0x51, 0xc0, 0x86, 0x82, 0x4a, 0x69, 0xaa, 0x40, 0x09, 0xa7, - 0xd3, 0xe6, 0x7f, 0xf2, 0xe0, 0xec, 0x85, 0xb1, 0x54, 0xb6, 0xb6, 0x7d, 0xb0, 0x8c, 0xbe, 0x84, - 0x3b, 0xc4, 0xfc, 0x33, 0x11, 0xa6, 0x0b, 0x85, 0xe9, 0xbd, 0x6c, 0x56, 0x1f, 0x65, 0xba, 0x9b, - 0x81, 0x93, 0x3e, 0xad, 0x53, 0xd0, 0x3e, 0xdd, 0x1c, 0xae, 0x92, 0x37, 0xbe, 0xa0, 0x13, 0xd8, - 0xe0, 0xc2, 0x1b, 0x51, 0xa9, 0x92, 0xf2, 0x62, 0xff, 0x31, 0x32, 0xff, 0x3e, 0x9f, 0x2f, 0x02, - 0xed, 0xdb, 0x9a, 0x44, 0xbb, 0xec, 0x03, 0x3d, 0x81, 0xbc, 0x20, 0xe7, 0x69, 0x1f, 0x99, 0xa9, - 0x7c, 0x4c, 0xce, 0xd5, 0x92, 0x0b, 0xc3, 0x40, 0xbf, 0x05, 0xf0, 0x03, 0x19, 0x11, 0xe5, 0x8d, - 0xa8, 0xb0, 0x27, 0x98, 0xb9, 0xc5, 0xee, 0x0c, 0xb5, 0xe4, 0x65, 0x81, 0x8d, 0x0e, 0xa1, 0xec, - 0x91, 0x54, 0x83, 0x85, 0x9b, 0x7f, 0xbc, 0xf6, 0xda, 0xd6, 0x45, 0x55, 0xbb, 0xb8, 0xbe, 0xda, - 0x2a, 0xa5, 0x16, 0x5c, 0xf2, 0x88, 0xd5, 0xe4, 0x21, 0x6c, 0xe8, 0x1f, 0xb2, 0x81, 0x4f, 0xcf, - 0x49, 0x1c, 0xaa, 0xe4, 0xec, 0x6f, 0xa8, 0x15, 0xba, 0xbb, 0xef, 0x5a, 0x9c, 0x8d, 0xab, 0xa2, - 0x16, 0x6c, 0xe8, 0xf7, 0x70, 0x87, 0x32, 0x4f, 0x4c, 0x8d, 0x02, 0xd3, 0x08, 0x4b, 0x37, 0x6f, - 0xb6, 0x37, 0x03, 0x2f, 0x6d, 0xb6, 0x4a, 0xdf, 0xb0, 0x37, 0x03, 0x80, 0xa4, 0xfa, 0x7e, 0x58, - 0xfd, 0x21, 0xc8, 0xfb, 0x44, 0x11, 0x23, 0xb9, 0x0a, 0x36, 0x63, 0xbd, 0x54, 0xb2, 0xe8, 0xff, - 0x7d, 0xa9, 0x8e, 0xfb, 0xfa, 0x9b, 0xfa, 0xca, 0x3f, 0xbf, 0xa9, 0xaf, 0xfc, 0xe9, 0xba, 0x9e, - 0x7b, 0x7d, 0x5d, 0xcf, 0xfd, 0xfd, 0xba, 0x9e, 0xfb, 0xd7, 0x75, 0x3d, 0x77, 0x56, 0x30, 0xed, - 0xd1, 0x4f, 0xfe, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x90, 0x54, 0x3b, 0xfc, 0x57, 0x12, 0x00, 0x00, + 0x15, 0xb7, 0x6c, 0x59, 0x7f, 0xde, 0xc8, 0x89, 0xdc, 0x24, 0x61, 0xac, 0xb0, 0xb2, 0xa2, 0x0d, + 0xc1, 0xcb, 0x16, 0x72, 0x61, 0xa8, 0x25, 0x4b, 0x58, 0x40, 0xb2, 0xb4, 0x8e, 0x31, 0x76, 0x54, + 0x6d, 0x6f, 0x20, 0x27, 0x55, 0x7b, 0xa6, 0x2d, 0x4d, 0x79, 0xd4, 0x3d, 0xf4, 0xf4, 0x68, 0x4b, + 0x37, 0x8e, 0x5b, 0xb9, 0x72, 0x76, 0x71, 0xe0, 0xcb, 0xe4, 0x48, 0x71, 0xe2, 0xe4, 0x62, 0xf5, + 0x09, 0xa8, 0xe2, 0x03, 0x40, 0x75, 0x4f, 0x8f, 0x34, 0x4a, 0xc6, 0x49, 0xaa, 0xc8, 0xde, 0xba, + 0xdf, 0xfc, 0x7e, 0xaf, 0x5f, 0xbf, 0xfe, 0x75, 0xbf, 0x37, 0x60, 0x85, 0x01, 0x75, 0xc2, 0x56, + 0x20, 0xb8, 0xe4, 0x08, 0xb9, 0xdc, 0xb9, 0xa4, 0xa2, 0x15, 0x7e, 0x4d, 0xc4, 0xf8, 0xd2, 0x93, + 0xad, 0xc9, 0x4f, 0x6b, 0x96, 0x9c, 0x06, 0xd4, 0x00, 0x6a, 0x77, 0x86, 0x7c, 0xc8, 0xf5, 0x70, + 0x57, 0x8d, 0x8c, 0xb5, 0x3e, 0xe4, 0x7c, 0xe8, 0xd3, 0x5d, 0x3d, 0x3b, 0x8f, 0x2e, 0x76, 0xdd, + 0x48, 0x10, 0xe9, 0x71, 0x66, 0xbe, 0x6f, 0xbd, 0xfe, 0x9d, 0xb0, 0x69, 0xfc, 0xa9, 0x79, 0x95, + 0x87, 0xd2, 0x09, 0x77, 0xe9, 0x69, 0x40, 0x1d, 0x74, 0x00, 0x16, 0x61, 0x8c, 0x4b, 0xcd, 0x0d, + 0xed, 0x5c, 0x23, 0xb7, 0x63, 0xed, 0x6d, 0xb7, 0xde, 0x0c, 0xaa, 0xd5, 0x5e, 0xc0, 0x3a, 0xf9, + 0x57, 0xd7, 0xdb, 0x2b, 0x38, 0xcd, 0x44, 0xbf, 0x81, 0x8a, 0x4b, 0x43, 0x4f, 0x50, 0x77, 0x20, + 0xb8, 0x4f, 0xed, 0xd5, 0x46, 0x6e, 0xe7, 0xd6, 0xde, 0x0f, 0xb2, 0x3c, 0xa9, 0xc5, 0x31, 0xf7, + 0x29, 0xb6, 0x0c, 0x43, 0x4d, 0xd0, 0x01, 0xc0, 0x98, 0x8e, 0xcf, 0xa9, 0x08, 0x47, 0x5e, 0x60, + 0xaf, 0x69, 0xfa, 0x8f, 0x6e, 0xa2, 0xab, 0xd8, 0x5b, 0xc7, 0x73, 0x38, 0x4e, 0x51, 0xd1, 0x31, + 0x54, 0xc8, 0x84, 0x78, 0x3e, 0x39, 0xf7, 0x7c, 0x4f, 0x4e, 0xed, 0xbc, 0x76, 0xf5, 0xc9, 0x5b, + 0x5d, 0xb5, 0x53, 0x04, 0xbc, 0x44, 0x6f, 0xba, 0x00, 0x8b, 0x85, 0xd0, 0x23, 0x28, 0xf6, 0x7b, + 0x27, 0xdd, 0xc3, 0x93, 0x83, 0xea, 0x4a, 0x6d, 0xeb, 0xe5, 0x55, 0xe3, 0xae, 0xf2, 0xb1, 0x00, + 0xf4, 0x29, 0x73, 0x3d, 0x36, 0x44, 0x3b, 0x50, 0x6a, 0xef, 0xef, 0xf7, 0xfa, 0x67, 0xbd, 0x6e, + 0x35, 0x57, 0xab, 0xbd, 0xbc, 0x6a, 0xdc, 0x5b, 0x06, 0xb6, 0x1d, 0x87, 0x06, 0x92, 0xba, 0xb5, + 0xfc, 0x37, 0x7f, 0xab, 0xaf, 0x34, 0xbf, 0xc9, 0x41, 0x25, 0x1d, 0x04, 0x7a, 0x04, 0x85, 0xf6, + 0xfe, 0xd9, 0xe1, 0xf3, 0x5e, 0x75, 0x65, 0x41, 0x4f, 0x23, 0xda, 0x8e, 0xf4, 0x26, 0x14, 0x3d, + 0x84, 0xf5, 0x7e, 0xfb, 0xab, 0xd3, 0x5e, 0x35, 0xb7, 0x08, 0x27, 0x0d, 0xeb, 0x93, 0x28, 0xd4, + 0xa8, 0x2e, 0x6e, 0x1f, 0x9e, 0x54, 0x57, 0xb3, 0x51, 0x5d, 0x41, 0x3c, 0x66, 0x42, 0xf9, 0x6b, + 0x1e, 0xac, 0x53, 0x2a, 0x26, 0x9e, 0xf3, 0x81, 0x25, 0xf2, 0x19, 0xe4, 0x25, 0x09, 0x2f, 0xb5, + 0x34, 0xac, 0x6c, 0x69, 0x9c, 0x91, 0xf0, 0x52, 0x2d, 0x6a, 0xe8, 0x1a, 0xaf, 0x94, 0x21, 0x68, + 0xe0, 0x7b, 0x0e, 0x91, 0xd4, 0xd5, 0xca, 0xb0, 0xf6, 0x7e, 0x98, 0xc5, 0xc6, 0x73, 0x94, 0x89, + 0xff, 0xe9, 0x0a, 0x4e, 0x51, 0xd1, 0x13, 0x28, 0x0c, 0x7d, 0x7e, 0x4e, 0x7c, 0xad, 0x09, 0x6b, + 0xef, 0x41, 0x96, 0x93, 0x03, 0x8d, 0x58, 0x38, 0x30, 0x14, 0xf4, 0x18, 0x0a, 0x51, 0xe0, 0x12, + 0x49, 0xed, 0x82, 0x26, 0x37, 0xb2, 0xc8, 0x5f, 0x69, 0xc4, 0x3e, 0x67, 0x17, 0xde, 0x10, 0x1b, + 0x3c, 0x3a, 0x82, 0x12, 0xa3, 0xf2, 0x6b, 0x2e, 0x2e, 0x43, 0xbb, 0xd8, 0x58, 0xdb, 0xb1, 0xf6, + 0x3e, 0xcd, 0x14, 0x63, 0x8c, 0x69, 0x4b, 0x49, 0x9c, 0xd1, 0x98, 0x32, 0x19, 0xbb, 0xe9, 0xac, + 0xda, 0x39, 0x3c, 0x77, 0x80, 0x7e, 0x05, 0x25, 0xca, 0xdc, 0x80, 0x7b, 0x4c, 0xda, 0xa5, 0x9b, + 0x03, 0xe9, 0x19, 0x8c, 0x4a, 0x26, 0x9e, 0x33, 0x14, 0x5b, 0x70, 0xdf, 0x3f, 0x27, 0xce, 0xa5, + 0x5d, 0x7e, 0xcf, 0x6d, 0xcc, 0x19, 0x9d, 0x02, 0xe4, 0xc7, 0xdc, 0xa5, 0xcd, 0x5d, 0xd8, 0x7c, + 0x23, 0xd5, 0xa8, 0x06, 0x25, 0x93, 0xea, 0x58, 0x23, 0x79, 0x3c, 0x9f, 0x37, 0x6f, 0xc3, 0xc6, + 0x52, 0x5a, 0x9b, 0xff, 0xc8, 0x43, 0x29, 0x39, 0x6b, 0xd4, 0x86, 0xb2, 0xc3, 0x99, 0x24, 0x1e, + 0xa3, 0xc2, 0xc8, 0x2b, 0xf3, 0x64, 0xf6, 0x13, 0x90, 0x62, 0x3d, 0x5d, 0xc1, 0x0b, 0x16, 0xfa, + 0x12, 0xca, 0x82, 0x86, 0x3c, 0x12, 0x0e, 0x0d, 0x8d, 0xbe, 0x76, 0xb2, 0x15, 0x12, 0x83, 0x30, + 0xfd, 0x53, 0xe4, 0x09, 0xaa, 0xb2, 0x1c, 0xe2, 0x05, 0x15, 0x3d, 0x81, 0xa2, 0xa0, 0xa1, 0x24, + 0x42, 0xbe, 0x4d, 0x22, 0x38, 0x86, 0xf4, 0xb9, 0xef, 0x39, 0x53, 0x9c, 0x30, 0xd0, 0x13, 0x28, + 0x07, 0x3e, 0x71, 0xb4, 0x57, 0x7b, 0x5d, 0xd3, 0x3f, 0xca, 0xa2, 0xf7, 0x13, 0x10, 0x5e, 0xe0, + 0xd1, 0xe7, 0x00, 0x3e, 0x1f, 0x0e, 0x5c, 0xe1, 0x4d, 0xa8, 0x30, 0x12, 0xab, 0x65, 0xb1, 0xbb, + 0x1a, 0x81, 0xcb, 0x3e, 0x1f, 0xc6, 0x43, 0x74, 0xf0, 0x7f, 0xe9, 0x2b, 0xa5, 0xad, 0x23, 0x00, + 0x32, 0xff, 0x6a, 0xd4, 0xf5, 0xc9, 0x7b, 0xb9, 0x32, 0x27, 0x92, 0xa2, 0xa3, 0x07, 0x50, 0xb9, + 0xe0, 0xc2, 0xa1, 0x03, 0x73, 0x6b, 0xca, 0x5a, 0x13, 0x96, 0xb6, 0xc5, 0xfa, 0x42, 0x1d, 0x28, + 0x0e, 0x29, 0xa3, 0xc2, 0x73, 0x6c, 0xd0, 0x8b, 0x3d, 0xca, 0xbc, 0x90, 0x31, 0x04, 0x47, 0x4c, + 0x7a, 0x63, 0x6a, 0x56, 0x4a, 0x88, 0x9d, 0x32, 0x14, 0x45, 0xfc, 0xa5, 0xf9, 0x47, 0x40, 0x6f, + 0x62, 0x11, 0x82, 0xfc, 0xa5, 0xc7, 0x5c, 0x2d, 0xac, 0x32, 0xd6, 0x63, 0xd4, 0x82, 0x62, 0x40, + 0xa6, 0x3e, 0x27, 0xae, 0x11, 0xcb, 0x9d, 0x56, 0x5c, 0x2f, 0x5b, 0x49, 0xbd, 0x6c, 0xb5, 0xd9, + 0x14, 0x27, 0xa0, 0xe6, 0x11, 0xdc, 0xcd, 0xdc, 0x32, 0xda, 0x83, 0xca, 0x5c, 0x84, 0x03, 0xcf, + 0x2c, 0xd2, 0xb9, 0x3d, 0xbb, 0xde, 0xb6, 0xe6, 0x6a, 0x3d, 0xec, 0x62, 0x6b, 0x0e, 0x3a, 0x74, + 0x9b, 0x7f, 0x29, 0xc3, 0xc6, 0x92, 0x94, 0xd1, 0x1d, 0x58, 0xf7, 0xc6, 0x64, 0x48, 0x4d, 0x8c, + 0xf1, 0x04, 0xf5, 0xa0, 0xe0, 0x93, 0x73, 0xea, 0x2b, 0x41, 0xab, 0x43, 0xfd, 0xc9, 0x3b, 0xef, + 0x44, 0xeb, 0xf7, 0x1a, 0xdf, 0x63, 0x52, 0x4c, 0xb1, 0x21, 0x23, 0x1b, 0x8a, 0x0e, 0x1f, 0x8f, + 0x09, 0x53, 0x4f, 0xe7, 0xda, 0x4e, 0x19, 0x27, 0x53, 0x95, 0x19, 0x22, 0x86, 0xa1, 0x9d, 0xd7, + 0x66, 0x3d, 0x46, 0x55, 0x58, 0xa3, 0x6c, 0x62, 0xaf, 0x6b, 0x93, 0x1a, 0x2a, 0x8b, 0xeb, 0xc5, + 0x8a, 0x2c, 0x63, 0x35, 0x54, 0xbc, 0x28, 0xa4, 0xc2, 0x2e, 0xc6, 0x19, 0x55, 0x63, 0xf4, 0x0b, + 0x28, 0x8c, 0x79, 0xc4, 0x64, 0x68, 0x97, 0x74, 0xb0, 0x5b, 0x59, 0xc1, 0x1e, 0x2b, 0x84, 0x79, + 0xda, 0x0d, 0x1c, 0xf5, 0x60, 0x33, 0x94, 0x3c, 0x18, 0x0c, 0x05, 0x71, 0xe8, 0x20, 0xa0, 0xc2, + 0xe3, 0xae, 0x79, 0x9a, 0xb6, 0xde, 0x38, 0x94, 0xae, 0x69, 0x72, 0xf0, 0x6d, 0xc5, 0x39, 0x50, + 0x94, 0xbe, 0x66, 0xa0, 0x3e, 0x54, 0x82, 0xc8, 0xf7, 0x07, 0x3c, 0x88, 0xab, 0x54, 0xac, 0xa7, + 0xf7, 0x48, 0x59, 0x3f, 0xf2, 0xfd, 0x67, 0x31, 0x09, 0x5b, 0xc1, 0x62, 0x82, 0xee, 0x41, 0x61, + 0x28, 0x78, 0x14, 0x84, 0xb6, 0xa5, 0x93, 0x61, 0x66, 0xe8, 0x0b, 0x28, 0x86, 0xd4, 0x11, 0x54, + 0x86, 0x76, 0x45, 0x6f, 0xf5, 0xe3, 0xac, 0x45, 0x4e, 0x35, 0x04, 0xd3, 0x0b, 0x2a, 0x28, 0x73, + 0x28, 0x4e, 0x38, 0x68, 0x0b, 0xd6, 0xa4, 0x9c, 0xda, 0x1b, 0x8d, 0xdc, 0x4e, 0xa9, 0x53, 0x9c, + 0x5d, 0x6f, 0xaf, 0x9d, 0x9d, 0xbd, 0xc0, 0xca, 0xa6, 0x5e, 0xd0, 0x11, 0x0f, 0x25, 0x23, 0x63, + 0x6a, 0xdf, 0xd2, 0xb9, 0x9d, 0xcf, 0xd1, 0x0b, 0x00, 0x97, 0x85, 0x03, 0x47, 0x5f, 0x59, 0xfb, + 0xb6, 0xde, 0xdd, 0xa7, 0xef, 0xde, 0x5d, 0xf7, 0xe4, 0xd4, 0x54, 0x91, 0x8d, 0xd9, 0xf5, 0x76, + 0x79, 0x3e, 0xc5, 0x65, 0x97, 0x85, 0xf1, 0x10, 0x75, 0xc0, 0x1a, 0x51, 0xe2, 0xcb, 0x91, 0x33, + 0xa2, 0xce, 0xa5, 0x5d, 0xbd, 0xb9, 0x2c, 0x3c, 0xd5, 0x30, 0xe3, 0x21, 0x4d, 0x52, 0x0a, 0x56, + 0xa1, 0x86, 0xf6, 0xa6, 0xce, 0x55, 0x3c, 0x41, 0x1f, 0x01, 0xf0, 0x80, 0xb2, 0x41, 0x28, 0x5d, + 0x8f, 0xd9, 0x48, 0x6d, 0x19, 0x97, 0x95, 0xe5, 0x54, 0x19, 0xd0, 0x7d, 0xf5, 0x68, 0x13, 0x77, + 0xc0, 0x99, 0x3f, 0xb5, 0xbf, 0xa7, 0xbf, 0x96, 0x94, 0xe1, 0x19, 0xf3, 0xa7, 0x68, 0x1b, 0x2c, + 0xad, 0x8b, 0xd0, 0x1b, 0x32, 0xe2, 0xdb, 0x77, 0x74, 0x3e, 0x40, 0x99, 0x4e, 0xb5, 0x45, 0x9d, + 0x43, 0x9c, 0x8d, 0xd0, 0xbe, 0x7b, 0xf3, 0x39, 0x98, 0x60, 0x17, 0xe7, 0x60, 0x38, 0xe8, 0xd7, + 0x00, 0x81, 0xf0, 0x26, 0x9e, 0x4f, 0x87, 0x34, 0xb4, 0xef, 0xe9, 0x4d, 0xd7, 0x33, 0x5f, 0xeb, + 0x39, 0x0a, 0xa7, 0x18, 0xb5, 0xcf, 0xc1, 0x4a, 0xdd, 0x36, 0x75, 0x4b, 0x2e, 0xe9, 0xd4, 0x5c, + 0x60, 0x35, 0x54, 0x29, 0x99, 0x10, 0x3f, 0x8a, 0x3b, 0xe1, 0x32, 0x8e, 0x27, 0xbf, 0x5c, 0x7d, + 0x9c, 0xab, 0xed, 0x81, 0x95, 0x52, 0x1d, 0xfa, 0x18, 0x36, 0x04, 0x1d, 0x7a, 0xa1, 0x14, 0xd3, + 0x01, 0x89, 0xe4, 0xc8, 0xfe, 0xad, 0x26, 0x54, 0x12, 0x63, 0x3b, 0x92, 0xa3, 0xda, 0x00, 0x16, + 0x87, 0x87, 0x1a, 0x60, 0x29, 0x51, 0x84, 0x54, 0x4c, 0xa8, 0x50, 0xd5, 0x56, 0xe5, 0x3c, 0x6d, + 0x52, 0xe2, 0x0d, 0x29, 0x11, 0xce, 0x48, 0xbf, 0x1d, 0x65, 0x6c, 0x66, 0xea, 0x31, 0x48, 0x6e, + 0x88, 0x79, 0x0c, 0xcc, 0xb4, 0xf9, 0x9f, 0x1c, 0x54, 0xd2, 0x4d, 0x03, 0xda, 0x8f, 0x8b, 0xbd, + 0xde, 0xd2, 0xad, 0xbd, 0xdd, 0x77, 0x35, 0x19, 0xba, 0xb4, 0xfa, 0x91, 0x72, 0x76, 0xac, 0xfa, + 0x7b, 0x4d, 0x46, 0x3f, 0x87, 0xf5, 0x80, 0x0b, 0x99, 0x3c, 0x61, 0xd9, 0x09, 0xe6, 0x22, 0x29, + 0x45, 0x31, 0xb8, 0x39, 0x82, 0x5b, 0xcb, 0xde, 0xd0, 0x43, 0x58, 0x7b, 0x7e, 0xd8, 0xaf, 0xae, + 0xd4, 0xee, 0xbf, 0xbc, 0x6a, 0x7c, 0x7f, 0xf9, 0xe3, 0x73, 0x4f, 0xc8, 0x88, 0xf8, 0x87, 0x7d, + 0xf4, 0x63, 0x58, 0xef, 0x9e, 0x9c, 0x62, 0x5c, 0xcd, 0xd5, 0xb6, 0x5f, 0x5e, 0x35, 0xee, 0x2f, + 0xe3, 0xd4, 0x27, 0x1e, 0x31, 0x17, 0xf3, 0xf3, 0x79, 0xaf, 0xfb, 0xef, 0x55, 0xb0, 0xcc, 0xcb, + 0xfe, 0xa1, 0x7f, 0x87, 0x36, 0xe2, 0x52, 0x9e, 0x5c, 0xd9, 0xd5, 0x77, 0x56, 0xf4, 0x4a, 0x4c, + 0x30, 0x67, 0xfc, 0x00, 0x2a, 0x5e, 0x30, 0xf9, 0x6c, 0x40, 0x19, 0x39, 0xf7, 0x4d, 0xdb, 0x5b, + 0xc2, 0x96, 0xb2, 0xf5, 0x62, 0x93, 0x7a, 0x2f, 0x3c, 0x26, 0xa9, 0x60, 0xa6, 0xa1, 0x2d, 0xe1, + 0xf9, 0x1c, 0x7d, 0x01, 0x79, 0x2f, 0x20, 0x63, 0xd3, 0x86, 0x64, 0xee, 0xe0, 0xb0, 0xdf, 0x3e, + 0x36, 0x1a, 0xec, 0x94, 0x66, 0xd7, 0xdb, 0x79, 0x65, 0xc0, 0x9a, 0x86, 0xea, 0x49, 0x27, 0xa0, + 0x56, 0xd2, 0x6f, 0x7f, 0x09, 0xa7, 0x2c, 0x4a, 0x47, 0x1e, 0x1b, 0x0a, 0x1a, 0x86, 0xba, 0x0a, + 0x94, 0x70, 0x32, 0x55, 0xf7, 0x36, 0xde, 0xf1, 0xe0, 0x42, 0xf0, 0xb1, 0x6e, 0x22, 0xca, 0x18, + 0x62, 0xd3, 0x97, 0x82, 0x8f, 0x9b, 0xff, 0xcd, 0x83, 0xb5, 0xef, 0x47, 0xa1, 0x34, 0xc5, 0xef, + 0x83, 0xa5, 0xfc, 0x05, 0x6c, 0x12, 0xfd, 0x53, 0x45, 0x98, 0xaa, 0x24, 0xba, 0x39, 0x33, 0x69, + 0x7f, 0x98, 0xe9, 0x6e, 0x0e, 0x8e, 0x1b, 0xb9, 0x4e, 0x41, 0xf9, 0xb4, 0x73, 0xb8, 0x4a, 0x5e, + 0xfb, 0x82, 0x4e, 0x61, 0x83, 0x0b, 0x67, 0x44, 0x43, 0x19, 0xd7, 0x1f, 0xf3, 0x13, 0x92, 0xf9, + 0x7b, 0xfa, 0x2c, 0x0d, 0x34, 0x8f, 0x6f, 0x1c, 0xed, 0xb2, 0x0f, 0xf4, 0x18, 0xf2, 0x82, 0x5c, + 0x24, 0x8d, 0x66, 0xe6, 0xd5, 0xc0, 0xe4, 0x42, 0x2e, 0xb9, 0xd0, 0x0c, 0xf4, 0x3b, 0x00, 0xd7, + 0x0b, 0x03, 0x22, 0x9d, 0x11, 0x15, 0xe6, 0x88, 0x33, 0xb7, 0xd8, 0x9d, 0xa3, 0x96, 0xbc, 0xa4, + 0xd8, 0xe8, 0x08, 0xca, 0x0e, 0x49, 0x44, 0x5a, 0xb8, 0xf9, 0xcf, 0x6c, 0xbf, 0x6d, 0x5c, 0x54, + 0x95, 0x8b, 0xd9, 0xf5, 0x76, 0x29, 0xb1, 0xe0, 0x92, 0x43, 0x8c, 0x68, 0x8f, 0x60, 0x43, 0xfd, + 0xb1, 0x0d, 0x5c, 0x7a, 0x41, 0x22, 0x5f, 0xc6, 0xe2, 0xb8, 0xa1, 0x98, 0xa8, 0xf6, 0xbf, 0x6b, + 0x70, 0x26, 0xae, 0x8a, 0x4c, 0xd9, 0xd0, 0x1f, 0x60, 0x93, 0x32, 0x47, 0x4c, 0xb5, 0x44, 0x93, + 0x08, 0x4b, 0x37, 0x6f, 0xb6, 0x37, 0x07, 0x2f, 0x6d, 0xb6, 0x4a, 0x5f, 0xb3, 0x37, 0x3d, 0x80, + 0xb8, 0x3c, 0x7f, 0x58, 0xfd, 0x21, 0xc8, 0xbb, 0x44, 0x12, 0x2d, 0xb9, 0x0a, 0xd6, 0x63, 0xb5, + 0x54, 0xbc, 0xe8, 0x77, 0xbe, 0x54, 0xc7, 0x7e, 0xf5, 0x6d, 0x7d, 0xe5, 0x9f, 0xdf, 0xd6, 0x57, + 0xfe, 0x3c, 0xab, 0xe7, 0x5e, 0xcd, 0xea, 0xb9, 0xbf, 0xcf, 0xea, 0xb9, 0x7f, 0xcd, 0xea, 0xb9, + 0xf3, 0x82, 0xee, 0x9f, 0x7e, 0xf6, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xb7, 0x5c, 0xc9, + 0x78, 0x12, 0x00, 0x00, } diff --git a/api/specs.proto b/api/specs.proto index 0b65afa929..ba842660f2 100644 --- a/api/specs.proto +++ b/api/specs.proto @@ -341,6 +341,11 @@ message NetworkSpec { // swarm internally created only and it was identified by the name // "ingress" and the label "com.docker.swarm.internal": "true". bool ingress = 7; + + // ConfigFrom indicates that the network specific configuration + // for this network will be provided via another network, locally + // on the node where this network is being plumbed. + string config_from = 8; } // ClusterSpec specifies global cluster settings. diff --git a/manager/allocator/networkallocator/drivers_network_linux.go b/manager/allocator/networkallocator/drivers_network_linux.go new file mode 100644 index 0000000000..a896162d2d --- /dev/null +++ b/manager/allocator/networkallocator/drivers_network_linux.go @@ -0,0 +1,19 @@ +package networkallocator + +import ( + "github.com/docker/libnetwork/drivers/bridge/brmanager" + "github.com/docker/libnetwork/drivers/ipvlan/ivmanager" + "github.com/docker/libnetwork/drivers/macvlan/mvmanager" + "github.com/docker/libnetwork/drivers/overlay/ovmanager" + "github.com/docker/libnetwork/drivers/remote" +) + +func getInitializers() []initializer { + return []initializer{ + {remote.Init, "remote"}, + {ovmanager.Init, "overlay"}, + {mvmanager.Init, "macvlan"}, + {brmanager.Init, "bridge"}, + {ivmanager.Init, "ipvlan"}, + } +} diff --git a/manager/allocator/networkallocator/drivers_network.go b/manager/allocator/networkallocator/drivers_network_windows.go similarity index 91% rename from manager/allocator/networkallocator/drivers_network.go rename to manager/allocator/networkallocator/drivers_network_windows.go index 59bb0d7da1..b8ad76b092 100644 --- a/manager/allocator/networkallocator/drivers_network.go +++ b/manager/allocator/networkallocator/drivers_network_windows.go @@ -1,5 +1,3 @@ -// +build linux windows - package networkallocator import ( diff --git a/manager/allocator/networkallocator/networkallocator.go b/manager/allocator/networkallocator/networkallocator.go index 0b47d3549f..0cca5d4983 100644 --- a/manager/allocator/networkallocator/networkallocator.go +++ b/manager/allocator/networkallocator/networkallocator.go @@ -3,6 +3,7 @@ package networkallocator import ( "fmt" "net" + "strings" "github.com/docker/docker/pkg/plugingetter" "github.com/docker/libnetwork/datastore" @@ -62,6 +63,18 @@ type network struct { // endpoints is a map of endpoint IP to the poolID from which it // was allocated. endpoints map[string]string + + // isNodeLocal indicates whether the scope of the network's resources + // is local to the node. If true, it means the resources can only be + // allocated locally by the node where the network will be deployed. + // In this the swarm manager will skip the allocations. + isNodeLocal bool +} + +type networkDriver struct { + driver driverapi.Driver + name string + capability *driverapi.Capability } type initializer struct { @@ -110,22 +123,38 @@ func (na *NetworkAllocator) Allocate(n *api.Network) error { return fmt.Errorf("network %s already allocated", n.ID) } - pools, err := na.allocatePools(n) + d, err := na.resolveDriver(n) if err != nil { - return errors.Wrapf(err, "failed allocating pools and gateway IP for network %s", n.ID) + return err } - if err := na.allocateDriverState(n); err != nil { - na.freePools(n, pools) - return errors.Wrapf(err, "failed while allocating driver state for network %s", n.ID) + nw := &network{ + nw: n, + endpoints: make(map[string]string), + isNodeLocal: d.capability.DataScope == datastore.LocalScope, } - na.networks[n.ID] = &network{ - nw: n, - pools: pools, - endpoints: make(map[string]string), + // No swarm-level allocation can be provided by the network driver for + // node-local networks. Only thing needed is populating the driver's name + // in the driver's state. + if nw.isNodeLocal { + n.DriverState = &api.Driver{ + Name: d.name, + } + } else { + nw.pools, err = na.allocatePools(n) + if err != nil { + return errors.Wrapf(err, "failed allocating pools and gateway IP for network %s", n.ID) + } + + if err := na.allocateDriverState(n); err != nil { + na.freePools(n, nw.pools) + return errors.Wrapf(err, "failed while allocating driver state for network %s", n.ID) + } } + na.networks[n.ID] = nw + return nil } @@ -141,11 +170,18 @@ func (na *NetworkAllocator) Deallocate(n *api.Network) error { return fmt.Errorf("could not get networker state for network %s", n.ID) } + // No swarm-level resource deallocation needed for node-local networks + if localNet.isNodeLocal { + delete(na.networks, n.ID) + return nil + } + if err := na.freeDriverState(n); err != nil { return errors.Wrapf(err, "failed to free driver state for network %s", n.ID) } delete(na.networks, n.ID) + return na.freePools(n, localNet.pools) } @@ -282,24 +318,32 @@ func (na *NetworkAllocator) IsTaskAllocated(t *api.Task) bool { } // To determine whether the task has its resources allocated, - // we just need to look at one network(in case of + // we just need to look at one global scope network (in case of // multi-network attachment). This is because we make sure we // allocate for every network or we allocate for none. - // If the network is not allocated, the task cannot be allocated. - localNet, ok := na.networks[t.Networks[0].Network.ID] - if !ok { - return false - } + // Find the first global scope network + for _, nAttach := range t.Networks { + // If the network is not allocated, the task cannot be allocated. + localNet, ok := na.networks[nAttach.Network.ID] + if !ok { + return false + } - // Addresses empty. Task is not allocated. - if len(t.Networks[0].Addresses) == 0 { - return false - } + // Nothing else to check for local scope network + if localNet.isNodeLocal { + continue + } - // The allocated IP address not found in local endpoint state. Not allocated. - if _, ok := localNet.endpoints[t.Networks[0].Addresses[0]]; !ok { - return false + // Addresses empty. Task is not allocated. + if len(nAttach.Addresses) == 0 { + return false + } + + // The allocated IP address not found in local endpoint state. Not allocated. + if _, ok := localNet.endpoints[nAttach.Addresses[0]]; !ok { + return false + } } return true @@ -445,6 +489,9 @@ func (na *NetworkAllocator) DeallocateNode(node *api.Node) error { // networks that a task is attached to. func (na *NetworkAllocator) AllocateTask(t *api.Task) error { for i, nAttach := range t.Networks { + if localNet := na.getNetwork(nAttach.Network.ID); localNet != nil && localNet.isNodeLocal { + continue + } if err := na.allocateNetworkIPs(nAttach); err != nil { if err := na.releaseEndpoints(t.Networks[:i]); err != nil { log.G(context.TODO()).WithError(err).Errorf("Failed to release IP addresses while rolling back allocation for task %s network %s", t.ID, nAttach.Network.ID) @@ -467,16 +514,20 @@ func (na *NetworkAllocator) DeallocateTask(t *api.Task) error { func (na *NetworkAllocator) releaseEndpoints(networks []*api.NetworkAttachment) error { for _, nAttach := range networks { - ipam, _, _, err := na.resolveIPAM(nAttach.Network) - if err != nil { - return errors.Wrapf(err, "failed to resolve IPAM while allocating") - } - localNet := na.getNetwork(nAttach.Network.ID) if localNet == nil { return fmt.Errorf("could not find network allocator state for network %s", nAttach.Network.ID) } + if localNet.isNodeLocal { + continue + } + + ipam, _, _, err := na.resolveIPAM(nAttach.Network) + if err != nil { + return errors.Wrapf(err, "failed to resolve IPAM while releasing") + } + // Do not fail and bail out if we fail to release IP // address here. Keep going and try releasing as many // addresses as possible. @@ -512,6 +563,10 @@ func (na *NetworkAllocator) allocateVIP(vip *api.Endpoint_VirtualIP) error { return errors.New("networkallocator: could not find local network state") } + if localNet.isNodeLocal { + return nil + } + // If this IP is already allocated in memory we don't need to // do anything. if _, ok := localNet.endpoints[vip.Addr]; ok { @@ -556,7 +611,9 @@ func (na *NetworkAllocator) deallocateVIP(vip *api.Endpoint_VirtualIP) error { if localNet == nil { return errors.New("networkallocator: could not find local network state") } - + if localNet.isNodeLocal { + return nil + } ipam, _, _, err := na.resolveIPAM(localNet.nw) if err != nil { return errors.Wrap(err, "failed to resolve IPAM while allocating") @@ -637,16 +694,16 @@ func (na *NetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment) e } func (na *NetworkAllocator) freeDriverState(n *api.Network) error { - d, _, err := na.resolveDriver(n) + d, err := na.resolveDriver(n) if err != nil { return err } - return d.NetworkFree(n.ID) + return d.driver.NetworkFree(n.ID) } func (na *NetworkAllocator) allocateDriverState(n *api.Network) error { - d, dName, err := na.resolveDriver(n) + d, err := na.resolveDriver(n) if err != nil { return err } @@ -691,14 +748,14 @@ func (na *NetworkAllocator) allocateDriverState(n *api.Network) error { ipv4Data = append(ipv4Data, data) } - ds, err := d.NetworkAllocate(n.ID, options, ipv4Data, nil) + ds, err := d.driver.NetworkAllocate(n.ID, options, ipv4Data, nil) if err != nil { return err } // Update network object with the obtained driver state. n.DriverState = &api.Driver{ - Name: dName, + Name: d.name, Options: ds, } @@ -706,7 +763,7 @@ func (na *NetworkAllocator) allocateDriverState(n *api.Network) error { } // Resolve network driver -func (na *NetworkAllocator) resolveDriver(n *api.Network) (driverapi.Driver, string, error) { +func (na *NetworkAllocator) resolveDriver(n *api.Network) (*networkDriver, error) { dName := DefaultDriver if n.Spec.DriverConfig != nil && n.Spec.DriverConfig.Name != "" { dName = n.Spec.DriverConfig.Name @@ -717,21 +774,16 @@ func (na *NetworkAllocator) resolveDriver(n *api.Network) (driverapi.Driver, str var err error err = na.loadDriver(dName) if err != nil { - return nil, "", err + return nil, err } d, drvcap = na.drvRegistry.Driver(dName) if d == nil { - return nil, "", fmt.Errorf("could not resolve network driver %s", dName) + return nil, fmt.Errorf("could not resolve network driver %s", dName) } - - } - - if drvcap.DataScope != datastore.GlobalScope { - return nil, "", fmt.Errorf("swarm can allocate network resources only for global scoped networks. network driver (%s) is scoped %s", dName, drvcap.DataScope) } - return d, dName, nil + return &networkDriver{driver: d, capability: drvcap, name: dName}, nil } func (na *NetworkAllocator) loadDriver(name string) error { @@ -934,3 +986,14 @@ func IsIngressNetworkNeeded(s *api.Service) bool { return false } + +// IsBuiltInDriver returns whether the passed driver is an internal network driver +func IsBuiltInDriver(name string) bool { + n := strings.ToLower(name) + for _, d := range getInitializers() { + if n == d.ntype { + return true + } + } + return false +} diff --git a/manager/controlapi/common.go b/manager/controlapi/common.go index 795fd936d1..56f0f8a00d 100644 --- a/manager/controlapi/common.go +++ b/manager/controlapi/common.go @@ -104,8 +104,16 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType return grpc.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required") } - if strings.ToLower(driver.Name) == networkallocator.DefaultDriver || strings.ToLower(driver.Name) == ipamapi.DefaultIPAM { - return nil + // First check against the known drivers + switch pluginType { + case ipamapi.PluginEndpointType: + if strings.ToLower(driver.Name) == ipamapi.DefaultIPAM { + return nil + } + default: + if networkallocator.IsBuiltInDriver(driver.Name) { + return nil + } } if pg == nil {