]> git.apps.os.sepia.ceph.com Git - ceph.git/blob
e1228f5ef90ac98921620c62b892577e0a57621b
[ceph.git] /
1 import { HttpParams } from '@angular/common/http';
2 import { Component, Input, OnInit, ViewChild } from '@angular/core';
3 import { AbstractControl, UntypedFormControl, Validators } from '@angular/forms';
4 import { ActivatedRoute, Router } from '@angular/router';
5
6 import { NgbActiveModal, NgbModalRef, NgbTypeahead } from '@ng-bootstrap/ng-bootstrap';
7 import { ListItem } from 'carbon-components-angular';
8 import _ from 'lodash';
9 import { forkJoin, merge, Observable, Subject, Subscription } from 'rxjs';
10 import { debounceTime, distinctUntilChanged, filter, map } from 'rxjs/operators';
11 import { Pool } from '~/app/ceph/pool/pool';
12 import { CreateRgwServiceEntitiesComponent } from '~/app/ceph/rgw/create-rgw-service-entities/create-rgw-service-entities.component';
13 import { RgwRealm, RgwZonegroup, RgwZone } from '~/app/ceph/rgw/models/rgw-multisite';
14
15 import { CephServiceService } from '~/app/shared/api/ceph-service.service';
16 import { HostService } from '~/app/shared/api/host.service';
17 import { PoolService } from '~/app/shared/api/pool.service';
18 import { RbdService } from '~/app/shared/api/rbd.service';
19 import { RgwMultisiteService } from '~/app/shared/api/rgw-multisite.service';
20 import { RgwRealmService } from '~/app/shared/api/rgw-realm.service';
21 import { RgwZoneService } from '~/app/shared/api/rgw-zone.service';
22 import { RgwZonegroupService } from '~/app/shared/api/rgw-zonegroup.service';
23 import { SelectMessages } from '~/app/shared/components/select/select-messages.model';
24 import { SelectOption } from '~/app/shared/components/select/select-option.model';
25 import {
26   ActionLabelsI18n,
27   TimerServiceInterval,
28   URLVerbs,
29   SSL_PROTOCOLS,
30   SSL_CIPHERS
31 } from '~/app/shared/constants/app.constants';
32 import { CdForm } from '~/app/shared/forms/cd-form';
33 import { CdFormBuilder } from '~/app/shared/forms/cd-form-builder';
34 import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
35 import { CdValidators } from '~/app/shared/forms/cd-validators';
36 import { FinishedTask } from '~/app/shared/models/finished-task';
37 import { Host } from '~/app/shared/models/host.interface';
38 import { CephServiceSpec } from '~/app/shared/models/service.interface';
39 import { ModalService } from '~/app/shared/services/modal.service';
40 import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
41 import { TimerService } from '~/app/shared/services/timer.service';
42
43 @Component({
44   selector: 'cd-service-form',
45   templateUrl: './service-form.component.html',
46   styleUrls: ['./service-form.component.scss']
47 })
48 export class ServiceFormComponent extends CdForm implements OnInit {
49   public sub = new Subscription();
50
51   readonly MDS_SVC_ID_PATTERN = /^[a-zA-Z_.-][a-zA-Z0-9_.-]*$/;
52   readonly SNMP_DESTINATION_PATTERN = /^[^\:]+:[0-9]/;
53   readonly SNMP_ENGINE_ID_PATTERN = /^[0-9A-Fa-f]{10,64}/g;
54   readonly INGRESS_SUPPORTED_SERVICE_TYPES = ['rgw', 'nfs'];
55   readonly SMB_CONFIG_URI_PATTERN = /^(http:|https:|rados:|rados:mon-config-key:)/;
56   readonly OAUTH2_ISSUER_URL_PATTERN = /^(https?:\/\/)?([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(:[0-9]{1,5})?(\/.*)?$/;
57   readonly SSL_CIPHERS_PATTERN = /^[a-zA-Z0-9\-:]+$/;
58   readonly DEFAULT_SSL_PROTOCOL_ITEM = [{ content: 'TLSv1.3', selected: true }];
59   @ViewChild(NgbTypeahead, { static: false })
60   typeahead: NgbTypeahead;
61
62   @Input() hiddenServices: string[] = [];
63
64   @Input() editing = false;
65
66   @Input() serviceName: string;
67
68   @Input() serviceType: string;
69
70   serviceForm: CdFormGroup;
71   action: string;
72   resource: string;
73   serviceTypes: string[] = [];
74   serviceIds: string[] = [];
75   hosts: any;
76   labels: string[];
77   labelClick = new Subject<string>();
78   labelFocus = new Subject<string>();
79   pools: Array<Pool>;
80   rbdPools: Array<Pool>;
81   services: Array<CephServiceSpec> = [];
82   pageURL: string;
83   serviceList: CephServiceSpec[];
84   multisiteInfo: object[] = [];
85   defaultRealmId = '';
86   defaultZonegroupId = '';
87   defaultZoneId = '';
88   realmList: RgwRealm[] = [];
89   zonegroupList: RgwZonegroup[] = [];
90   zoneList: RgwZone[] = [];
91   bsModalRef: NgbModalRef;
92   defaultZonegroup: RgwZonegroup;
93   showRealmCreationForm = false;
94   defaultsInfo: { defaultRealmName: string; defaultZonegroupName: string; defaultZoneName: string };
95   realmNames: string[];
96   zonegroupNames: string[];
97   zoneNames: string[];
98   smbFeaturesList = ['domain'];
99   currentURL: string;
100   port: number = 443;
101   sslProtocolsItems: Array<ListItem> = Object.values(SSL_PROTOCOLS).map((protocol) => ({
102     content: protocol,
103     selected: true
104   }));
105   sslCiphersItems: Array<ListItem> = Object.values(SSL_CIPHERS).map((cipher) => ({
106     content: cipher,
107     selected: false
108   }));
109   showMgmtGatewayMessage: boolean = false;
110
111   constructor(
112     public actionLabels: ActionLabelsI18n,
113     private cephServiceService: CephServiceService,
114     private formBuilder: CdFormBuilder,
115     private hostService: HostService,
116     private poolService: PoolService,
117     private rbdService: RbdService,
118     private router: Router,
119     private taskWrapperService: TaskWrapperService,
120     public timerService: TimerService,
121     public timerServiceVariable: TimerServiceInterval,
122     public rgwRealmService: RgwRealmService,
123     public rgwZonegroupService: RgwZonegroupService,
124     public rgwZoneService: RgwZoneService,
125     public rgwMultisiteService: RgwMultisiteService,
126     private route: ActivatedRoute,
127     public activeModal: NgbActiveModal,
128     public modalService: ModalService
129   ) {
130     super();
131     this.resource = $localize`service`;
132     this.hosts = {
133       options: [],
134       messages: new SelectMessages({
135         empty: $localize`There are no hosts.`,
136         filter: $localize`Filter hosts`
137       })
138     };
139     this.createForm();
140   }
141
142   createForm() {
143     this.serviceForm = this.formBuilder.group({
144       // Global
145       service_type: [null, [Validators.required]],
146       service_id: [
147         null,
148         [
149           CdValidators.composeIf(
150             {
151               service_type: 'mds'
152             },
153             [
154               Validators.required,
155               CdValidators.custom('mdsPattern', (value: string) => {
156                 if (_.isEmpty(value)) {
157                   return false;
158                 }
159                 return !this.MDS_SVC_ID_PATTERN.test(value);
160               })
161             ]
162           ),
163           CdValidators.requiredIf({
164             service_type: 'nfs'
165           }),
166           CdValidators.requiredIf({
167             service_type: 'iscsi'
168           }),
169           CdValidators.requiredIf({
170             service_type: 'nvmeof'
171           }),
172           CdValidators.requiredIf({
173             service_type: 'ingress'
174           }),
175           CdValidators.requiredIf({
176             service_type: 'smb'
177           }),
178           CdValidators.composeIf(
179             {
180               service_type: 'rgw'
181             },
182             [Validators.required]
183           ),
184           CdValidators.custom('uniqueName', (service_id: string) => {
185             return this.serviceIds && this.serviceIds.includes(service_id);
186           })
187         ]
188       ],
189       placement: ['hosts'],
190       label: [
191         null,
192         [
193           CdValidators.requiredIf({
194             placement: 'label',
195             unmanaged: false
196           })
197         ]
198       ],
199       hosts: [[]],
200       count: [null, [CdValidators.number(false)]],
201       unmanaged: [false],
202       // iSCSI
203       // NVMe/TCP
204       pool: [
205         null,
206         [
207           CdValidators.requiredIf({
208             service_type: 'iscsi'
209           }),
210           CdValidators.requiredIf({
211             service_type: 'nvmeof'
212           })
213         ]
214       ],
215       group: [
216         'default',
217         CdValidators.requiredIf({
218           service_type: 'nvmeof'
219         })
220       ],
221       enable_mtls: [false],
222       root_ca_cert: [
223         null,
224         [
225           CdValidators.composeIf(
226             {
227               service_type: 'nvmeof',
228               enable_mtls: true
229             },
230             [Validators.required]
231           )
232         ]
233       ],
234       client_cert: [
235         null,
236         [
237           CdValidators.composeIf(
238             {
239               service_type: 'nvmeof',
240               enable_mtls: true
241             },
242             [Validators.required]
243           )
244         ]
245       ],
246       client_key: [
247         null,
248         [
249           CdValidators.composeIf(
250             {
251               service_type: 'nvmeof',
252               enable_mtls: true
253             },
254             [Validators.required]
255           )
256         ]
257       ],
258       server_cert: [
259         null,
260         [
261           CdValidators.composeIf(
262             {
263               service_type: 'nvmeof',
264               enable_mtls: true
265             },
266             [Validators.required]
267           )
268         ]
269       ],
270       server_key: [
271         null,
272         [
273           CdValidators.composeIf(
274             {
275               service_type: 'nvmeof',
276               enable_mtls: true
277             },
278             [Validators.required]
279           )
280         ]
281       ],
282       // RGW
283       rgw_frontend_port: [null, [CdValidators.number(false)]],
284       realm_name: [null],
285       zonegroup_name: [null],
286       zone_name: [null],
287       // iSCSI
288       trusted_ip_list: [null],
289       api_port: [null, [CdValidators.number(false)]],
290       api_user: [
291         null,
292         [
293           CdValidators.requiredIf({
294             service_type: 'iscsi',
295             unmanaged: false
296           })
297         ]
298       ],
299       api_password: [
300         null,
301         [
302           CdValidators.requiredIf({
303             service_type: 'iscsi',
304             unmanaged: false
305           })
306         ]
307       ],
308       // smb
309       cluster_id: [
310         null,
311         [
312           CdValidators.requiredIf({
313             service_type: 'smb'
314           })
315         ]
316       ],
317       features: new CdFormGroup(
318         this.smbFeaturesList.reduce((acc: object, e) => {
319           acc[e] = new UntypedFormControl(false);
320           return acc;
321         }, {})
322       ),
323       config_uri: [
324         null,
325         [
326           CdValidators.composeIf(
327             {
328               service_type: 'smb'
329             },
330             [
331               Validators.required,
332               CdValidators.custom('configUriPattern', (value: string) => {
333                 if (_.isEmpty(value)) {
334                   return false;
335                 }
336                 return !this.SMB_CONFIG_URI_PATTERN.test(value);
337               })
338             ]
339           )
340         ]
341       ],
342       custom_dns: [null],
343       join_sources: [null],
344       user_sources: [null],
345       include_ceph_users: [null],
346       // Ingress
347       backend_service: [
348         null,
349         [
350           CdValidators.requiredIf({
351             service_type: 'ingress'
352           })
353         ]
354       ],
355       virtual_ip: [
356         null,
357         [
358           CdValidators.requiredIf({
359             service_type: 'ingress'
360           })
361         ]
362       ],
363       frontend_port: [
364         null,
365         [
366           CdValidators.number(false),
367           CdValidators.requiredIf({
368             service_type: 'ingress'
369           })
370         ]
371       ],
372       monitor_port: [
373         null,
374         [
375           CdValidators.number(false),
376           CdValidators.requiredIf({
377             service_type: 'ingress'
378           })
379         ]
380       ],
381       virtual_interface_networks: [null],
382       ssl_protocols: [this.DEFAULT_SSL_PROTOCOL_ITEM],
383       ssl_ciphers: [
384         null,
385         [
386           CdValidators.custom('invalidPattern', (ciphers: string) => {
387             if (_.isEmpty(ciphers)) {
388               return false;
389             }
390             return !this.SSL_CIPHERS_PATTERN.test(ciphers);
391           })
392         ]
393       ],
394       // RGW, Ingress & iSCSI
395       ssl: [false],
396       ssl_cert: [
397         '',
398         [
399           CdValidators.composeIf(
400             {
401               service_type: 'rgw',
402               unmanaged: false,
403               ssl: true
404             },
405             [Validators.required, CdValidators.pemCert()]
406           ),
407           CdValidators.composeIf(
408             {
409               service_type: 'iscsi',
410               unmanaged: false,
411               ssl: true
412             },
413             [Validators.required, CdValidators.sslCert()]
414           ),
415           CdValidators.composeIf(
416             {
417               service_type: 'ingress',
418               unmanaged: false,
419               ssl: true
420             },
421             [Validators.required, CdValidators.pemCert()]
422           ),
423           CdValidators.composeIf(
424             {
425               service_type: 'oauth2-proxy',
426               unmanaged: false,
427               ssl: true
428             },
429             [Validators.required, CdValidators.sslCert()]
430           ),
431           CdValidators.composeIf(
432             {
433               service_type: 'mgmt-gateway',
434               unmanaged: false,
435               ssl: false
436             },
437             [CdValidators.sslCert()]
438           )
439         ]
440       ],
441       ssl_key: [
442         '',
443         [
444           CdValidators.composeIf(
445             {
446               service_type: 'iscsi',
447               unmanaged: false,
448               ssl: true
449             },
450             [Validators.required, CdValidators.sslPrivKey()]
451           ),
452           CdValidators.composeIf(
453             {
454               service_type: 'oauth2-proxy',
455               unmanaged: false,
456               ssl: true
457             },
458             [Validators.required, CdValidators.sslPrivKey()]
459           ),
460           CdValidators.composeIf(
461             {
462               service_type: 'mgmt-gateway',
463               unmanaged: false,
464               ssl: false
465             },
466             [CdValidators.sslPrivKey()]
467           )
468         ]
469       ],
470       // mgmt-gateway
471       enable_auth: [null],
472       port: [443, [CdValidators.number(false)]],
473       // snmp-gateway
474       snmp_version: [
475         null,
476         [
477           CdValidators.requiredIf({
478             service_type: 'snmp-gateway'
479           })
480         ]
481       ],
482       snmp_destination: [
483         null,
484         {
485           validators: [
486             CdValidators.requiredIf({
487               service_type: 'snmp-gateway'
488             }),
489             CdValidators.custom('snmpDestinationPattern', (value: string) => {
490               if (_.isEmpty(value)) {
491                 return false;
492               }
493               return !this.SNMP_DESTINATION_PATTERN.test(value);
494             })
495           ]
496         }
497       ],
498       engine_id: [
499         null,
500         [
501           CdValidators.requiredIf({
502             service_type: 'snmp-gateway'
503           }),
504           CdValidators.custom('snmpEngineIdPattern', (value: string) => {
505             if (_.isEmpty(value)) {
506               return false;
507             }
508             return !this.SNMP_ENGINE_ID_PATTERN.test(value);
509           })
510         ]
511       ],
512       auth_protocol: [
513         'SHA',
514         [
515           CdValidators.requiredIf({
516             service_type: 'snmp-gateway'
517           })
518         ]
519       ],
520       privacy_protocol: [null],
521       snmp_community: [
522         null,
523         [
524           CdValidators.requiredIf({
525             snmp_version: 'V2c'
526           })
527         ]
528       ],
529       snmp_v3_auth_username: [
530         null,
531         [
532           CdValidators.requiredIf({
533             service_type: 'snmp-gateway'
534           })
535         ]
536       ],
537       snmp_v3_auth_password: [
538         null,
539         [
540           CdValidators.requiredIf({
541             service_type: 'snmp-gateway'
542           })
543         ]
544       ],
545       snmp_v3_priv_password: [
546         null,
547         [
548           CdValidators.requiredIf({
549             privacy_protocol: { op: '!empty' }
550           })
551         ]
552       ],
553       grafana_port: [null, [CdValidators.number(false)]],
554       grafana_admin_password: [null],
555       // oauth2-proxy
556       provider_display_name: [
557         'My OIDC provider',
558         [
559           CdValidators.requiredIf({
560             service_type: 'oauth2-proxy'
561           })
562         ]
563       ],
564       client_id: [
565         null,
566         [
567           CdValidators.requiredIf({
568             service_type: 'oauth2-proxy'
569           })
570         ]
571       ],
572       client_secret: [
573         null,
574         [
575           CdValidators.requiredIf({
576             service_type: 'oauth2-proxy'
577           })
578         ]
579       ],
580       oidc_issuer_url: [
581         null,
582         [
583           CdValidators.requiredIf({
584             service_type: 'oauth2-proxy'
585           }),
586           CdValidators.custom('validUrl', (url: string) => {
587             if (_.isEmpty(url)) {
588               return false;
589             }
590             return !this.OAUTH2_ISSUER_URL_PATTERN.test(url);
591           })
592         ]
593       ],
594       https_address: [null, [CdValidators.oauthAddressTest()]],
595       redirect_url: [null],
596       allowlist_domains: [null]
597     });
598   }
599
600   resolveRoute() {
601     if (this.router.url.includes('services/(modal:create')) {
602       this.pageURL = 'services';
603       this.route.params.subscribe((params: { type: string }) => {
604         if (params?.type) {
605           this.serviceType = params.type;
606           this.serviceForm.get('service_type').setValue(this.serviceType);
607         }
608       });
609     } else if (this.router.url.includes('services/(modal:edit')) {
610       this.editing = true;
611       this.pageURL = 'services';
612       this.route.params.subscribe((params: { type: string; name: string }) => {
613         this.serviceName = params.name;
614         this.serviceType = params.type;
615       });
616     }
617   }
618
619   ngOnInit(): void {
620     this.action = this.actionLabels.CREATE;
621     this.resolveRoute();
622
623     this.cephServiceService
624       .list(new HttpParams({ fromObject: { limit: -1, offset: 0 } }))
625       .observable.subscribe((services: CephServiceSpec[]) => {
626         this.serviceList = services;
627         this.services = services.filter((service: any) =>
628           this.INGRESS_SUPPORTED_SERVICE_TYPES.includes(service.service_type)
629         );
630       });
631
632     this.cephServiceService.getKnownTypes().subscribe((resp: Array<string>) => {
633       // Remove service types:
634       // osd       - This is deployed a different way.
635       // container - This should only be used in the CLI.
636       // promtail  - This is deprecated and replaced by alloy.
637       this.hiddenServices.push('osd', 'container', 'promtail');
638
639       this.serviceTypes = _.difference(resp, this.hiddenServices).sort();
640     });
641     this.hostService.getAllHosts().subscribe((resp: Host[]) => {
642       const options: SelectOption[] = [];
643       _.forEach(resp, (host: Host) => {
644         if (_.get(host, 'sources.orchestrator', false)) {
645           const option = new SelectOption(false, _.get(host, 'hostname'), '');
646           options.push(option);
647         }
648       });
649       this.hosts.options = [...options];
650     });
651     this.hostService.getLabels().subscribe((resp: string[]) => {
652       this.labels = resp;
653     });
654     this.poolService.getList().subscribe((resp: Pool[]) => {
655       this.pools = resp;
656       this.rbdPools = this.pools.filter(this.rbdService.isRBDPool);
657       if (!this.editing && this.serviceType) {
658         this.onServiceTypeChange(this.serviceType);
659       }
660     });
661
662     if (this.editing) {
663       this.action = this.actionLabels.EDIT;
664       this.disableForEditing(this.serviceType);
665       this.cephServiceService
666         .list(new HttpParams({ fromObject: { limit: -1, offset: 0 } }), this.serviceName)
667         .observable.subscribe((response: CephServiceSpec[]) => {
668           const formKeys = ['service_type', 'service_id', 'unmanaged'];
669           formKeys.forEach((keys) => {
670             this.serviceForm.get(keys).setValue(response[0][keys]);
671           });
672           if (!response[0]['unmanaged']) {
673             const placementKey = Object.keys(response[0]['placement'])[0];
674             let placementValue: string;
675             ['hosts', 'label'].indexOf(placementKey) >= 0
676               ? (placementValue = placementKey)
677               : (placementValue = 'hosts');
678             this.serviceForm.get('placement').setValue(placementValue);
679             this.serviceForm.get('count').setValue(response[0]['placement']['count']);
680             if (response[0]?.placement[placementValue]) {
681               this.serviceForm.get(placementValue).setValue(response[0]?.placement[placementValue]);
682             }
683           }
684           switch (this.serviceType) {
685             case 'iscsi':
686               const specKeys = ['pool', 'api_password', 'api_user', 'trusted_ip_list', 'api_port'];
687               specKeys.forEach((key) => {
688                 this.serviceForm.get(key).setValue(response[0].spec[key]);
689               });
690               this.serviceForm.get('ssl').setValue(response[0].spec?.api_secure);
691               if (response[0].spec?.api_secure) {
692                 this.serviceForm.get('ssl_cert').setValue(response[0].spec?.ssl_cert);
693                 this.serviceForm.get('ssl_key').setValue(response[0].spec?.ssl_key);
694               }
695               break;
696             case 'nvmeof':
697               this.serviceForm.get('pool').setValue(response[0].spec.pool);
698               this.serviceForm.get('group').setValue(response[0].spec.group);
699               this.serviceForm.get('enable_mtls').setValue(response[0].spec?.enable_auth);
700               this.serviceForm.get('root_ca_cert').setValue(response[0].spec?.root_ca_cert);
701               this.serviceForm.get('client_cert').setValue(response[0].spec?.client_cert);
702               this.serviceForm.get('client_key').setValue(response[0].spec?.client_key);
703               this.serviceForm.get('server_cert').setValue(response[0].spec?.server_cert);
704               this.serviceForm.get('server_key').setValue(response[0].spec?.server_key);
705               break;
706             case 'rgw':
707               this.serviceForm
708                 .get('rgw_frontend_port')
709                 .setValue(response[0].spec?.rgw_frontend_port);
710               this.setRgwFields(
711                 response[0].spec?.rgw_realm,
712                 response[0].spec?.rgw_zonegroup,
713                 response[0].spec?.rgw_zone
714               );
715               this.serviceForm.get('ssl').setValue(response[0].spec?.ssl);
716               if (response[0].spec?.ssl) {
717                 this.serviceForm
718                   .get('ssl_cert')
719                   .setValue(response[0].spec?.rgw_frontend_ssl_certificate);
720               }
721               break;
722             case 'ingress':
723               const ingressSpecKeys = [
724                 'backend_service',
725                 'virtual_ip',
726                 'frontend_port',
727                 'monitor_port',
728                 'virtual_interface_networks',
729                 'ssl'
730               ];
731               ingressSpecKeys.forEach((key) => {
732                 this.serviceForm.get(key).setValue(response[0].spec[key]);
733               });
734               if (response[0].spec?.ssl) {
735                 this.serviceForm.get('ssl_cert').setValue(response[0].spec?.ssl_cert);
736                 this.serviceForm.get('ssl_key').setValue(response[0].spec?.ssl_key);
737               }
738               break;
739             case 'mgmt-gateway':
740               let hrefSplitted = window.location.href.split(':');
741               this.currentURL = hrefSplitted[0] + hrefSplitted[1];
742               this.port = response[0].spec?.port;
743
744               if (response[0].spec?.ssl_protocols) {
745                 let selectedValues: Array<ListItem> = [];
746                 for (const value of response[0].spec.ssl_protocols) {
747                   selectedValues.push({ content: value, selected: true });
748                 }
749                 this.serviceForm.get('ssl_protocols').setValue(selectedValues);
750               }
751               if (response[0].spec?.ssl_ciphers) {
752                 this.serviceForm
753                   .get('ssl_ciphers')
754                   .setValue(response[0].spec?.ssl_ciphers.join(':'));
755               }
756               if (response[0].spec?.ssl_cert) {
757                 this.serviceForm.get('ssl_cert').setValue(response[0].spec.ssl_cert);
758               }
759               if (response[0].spec?.ssl_key) {
760                 this.serviceForm.get('ssl_key').setValue(response[0].spec.ssl_key);
761               }
762               if (response[0].spec?.enable_auth) {
763                 this.serviceForm.get('enable_auth').setValue(response[0].spec.enable_auth);
764               }
765               if (response[0].spec?.port) {
766                 this.serviceForm.get('port').setValue(response[0].spec.port);
767               }
768               break;
769             case 'smb':
770               const smbSpecKeys = [
771                 'cluster_id',
772                 'config_uri',
773                 'features',
774                 'join_sources',
775                 'user_sources',
776                 'custom_dns',
777                 'include_ceph_users'
778               ];
779               smbSpecKeys.forEach((key) => {
780                 if (key === 'features') {
781                   if (response[0].spec?.features) {
782                     response[0].spec.features.forEach((feature) => {
783                       this.serviceForm.get(`features.${feature}`).setValue(true);
784                     });
785                   }
786                 } else {
787                   this.serviceForm.get(key).setValue(response[0].spec[key]);
788                 }
789               });
790               break;
791             case 'snmp-gateway':
792               const snmpCommonSpecKeys = ['snmp_version', 'snmp_destination'];
793               snmpCommonSpecKeys.forEach((key) => {
794                 this.serviceForm.get(key).setValue(response[0].spec[key]);
795               });
796               if (this.serviceForm.getValue('snmp_version') === 'V3') {
797                 const snmpV3SpecKeys = [
798                   'engine_id',
799                   'auth_protocol',
800                   'privacy_protocol',
801                   'snmp_v3_auth_username',
802                   'snmp_v3_auth_password',
803                   'snmp_v3_priv_password'
804                 ];
805                 snmpV3SpecKeys.forEach((key) => {
806                   if (key !== null) {
807                     if (
808                       key === 'snmp_v3_auth_username' ||
809                       key === 'snmp_v3_auth_password' ||
810                       key === 'snmp_v3_priv_password'
811                     ) {
812                       this.serviceForm.get(key).setValue(response[0].spec['credentials'][key]);
813                     } else {
814                       this.serviceForm.get(key).setValue(response[0].spec[key]);
815                     }
816                   }
817                 });
818               } else {
819                 this.serviceForm
820                   .get('snmp_community')
821                   .setValue(response[0].spec['credentials']['snmp_community']);
822               }
823               break;
824             case 'grafana':
825               this.serviceForm.get('grafana_port').setValue(response[0].spec.port);
826               this.serviceForm
827                 .get('grafana_admin_password')
828                 .setValue(response[0].spec.initial_admin_password);
829               break;
830             case 'oauth2-proxy':
831               const oauth2SpecKeys = [
832                 'https_address',
833                 'provider_display_name',
834                 'client_id',
835                 'client_secret',
836                 'oidc_issuer_url',
837                 'redirect_url',
838                 'allowlist_domains'
839               ];
840               oauth2SpecKeys.forEach((key) => {
841                 this.serviceForm.get(key).setValue(response[0].spec[key]);
842               });
843               if (response[0].spec?.ssl) {
844                 this.serviceForm.get('ssl_cert').setValue(response[0].spec?.ssl_cert);
845                 this.serviceForm.get('ssl_key').setValue(response[0].spec?.ssl_key);
846               }
847           }
848         });
849     }
850     this.detectChanges();
851   }
852
853   detectChanges(): void {
854     const service_type = this.serviceForm.get('service_type');
855     if (service_type) {
856       service_type.valueChanges.subscribe((value) => {
857         if (value === 'mgmt-gateway') {
858           const port = this.serviceForm.get('port');
859           if (port) {
860             port.valueChanges.subscribe((_) => {
861               this.showMgmtGatewayMessage = true;
862             });
863           }
864           const ssl_protocols = this.serviceForm.get('ssl_protocols');
865           if (ssl_protocols) {
866             ssl_protocols.valueChanges.subscribe((_) => {
867               this.showMgmtGatewayMessage = true;
868             });
869           }
870           const ssl_ciphers = this.serviceForm.get('ssl_ciphers');
871           if (ssl_ciphers) {
872             ssl_ciphers.valueChanges.subscribe((_) => {
873               this.showMgmtGatewayMessage = true;
874             });
875           }
876         }
877       });
878     }
879   }
880
881   getDefaultsEntitiesForRgw(
882     defaultRealmId: string,
883     defaultZonegroupId: string,
884     defaultZoneId: string
885   ): { defaultRealmName: string; defaultZonegroupName: string; defaultZoneName: string } {
886     const defaultRealm = this.realmList.find((x: { id: string }) => x.id === defaultRealmId);
887     const defaultZonegroup = this.zonegroupList.find(
888       (x: { id: string }) => x.id === defaultZonegroupId
889     );
890     const defaultZone = this.zoneList.find((x: { id: string }) => x.id === defaultZoneId);
891     const defaultRealmName = defaultRealm !== undefined ? defaultRealm.name : null;
892     const defaultZonegroupName = defaultZonegroup !== undefined ? defaultZonegroup.name : 'default';
893     const defaultZoneName = defaultZone !== undefined ? defaultZone.name : 'default';
894     if (defaultZonegroupName === 'default' && !this.zonegroupNames.includes(defaultZonegroupName)) {
895       const defaultZonegroup = new RgwZonegroup();
896       defaultZonegroup.name = 'default';
897       this.zonegroupList.push(defaultZonegroup);
898     }
899     if (defaultZoneName === 'default' && !this.zoneNames.includes(defaultZoneName)) {
900       const defaultZone = new RgwZone();
901       defaultZone.name = 'default';
902       this.zoneList.push(defaultZone);
903     }
904     return {
905       defaultRealmName: defaultRealmName,
906       defaultZonegroupName: defaultZonegroupName,
907       defaultZoneName: defaultZoneName
908     };
909   }
910
911   getDefaultPlacementCount(serviceType: string) {
912     /**
913      * `defaults` from src/pybind/mgr/cephadm/module.py
914      */
915     switch (serviceType) {
916       case 'mon':
917         this.serviceForm.get('count').setValue(5);
918         break;
919       case 'mgr':
920       case 'mds':
921       case 'rgw':
922       case 'ingress':
923       case 'rbd-mirror':
924         this.serviceForm.get('count').setValue(2);
925         break;
926       case 'iscsi':
927       case 'cephfs-mirror':
928       case 'nfs':
929       case 'grafana':
930       case 'alertmanager':
931       case 'prometheus':
932       case 'loki':
933       case 'container':
934       case 'snmp-gateway':
935       case 'elastic-serach':
936       case 'jaeger-collector':
937       case 'jaeger-query':
938       case 'smb':
939       case 'oauth2-proxy':
940       case 'mgmt-gateway':
941         this.serviceForm.get('count').setValue(1);
942         break;
943       default:
944         this.serviceForm.get('count').setValue(null);
945     }
946   }
947
948   setRgwFields(realm_name?: string, zonegroup_name?: string, zone_name?: string) {
949     const observables = [
950       this.rgwRealmService.getAllRealmsInfo(),
951       this.rgwZonegroupService.getAllZonegroupsInfo(),
952       this.rgwZoneService.getAllZonesInfo()
953     ];
954     this.sub = forkJoin(observables).subscribe(
955       (multisiteInfo: [object, object, object]) => {
956         this.multisiteInfo = multisiteInfo;
957         this.realmList =
958           this.multisiteInfo[0] !== undefined && this.multisiteInfo[0].hasOwnProperty('realms')
959             ? this.multisiteInfo[0]['realms']
960             : [];
961         this.zonegroupList =
962           this.multisiteInfo[1] !== undefined && this.multisiteInfo[1].hasOwnProperty('zonegroups')
963             ? this.multisiteInfo[1]['zonegroups']
964             : [];
965         this.zoneList =
966           this.multisiteInfo[2] !== undefined && this.multisiteInfo[2].hasOwnProperty('zones')
967             ? this.multisiteInfo[2]['zones']
968             : [];
969         this.realmNames = this.realmList.map((realm) => {
970           return realm['name'];
971         });
972         this.zonegroupNames = this.zonegroupList.map((zonegroup) => {
973           return zonegroup['name'];
974         });
975         this.zoneNames = this.zoneList.map((zone) => {
976           return zone['name'];
977         });
978         this.defaultRealmId = multisiteInfo[0]['default_realm'];
979         this.defaultZonegroupId = multisiteInfo[1]['default_zonegroup'];
980         this.defaultZoneId = multisiteInfo[2]['default_zone'];
981         this.defaultsInfo = this.getDefaultsEntitiesForRgw(
982           this.defaultRealmId,
983           this.defaultZonegroupId,
984           this.defaultZoneId
985         );
986         if (!this.editing) {
987           this.serviceForm.get('realm_name').setValue(this.defaultsInfo['defaultRealmName']);
988           this.serviceForm
989             .get('zonegroup_name')
990             .setValue(this.defaultsInfo['defaultZonegroupName']);
991           this.serviceForm.get('zone_name').setValue(this.defaultsInfo['defaultZoneName']);
992         } else {
993           if (realm_name && !this.realmNames.includes(realm_name)) {
994             const realm = new RgwRealm();
995             realm.name = realm_name;
996             this.realmList.push(realm);
997           }
998           if (zonegroup_name && !this.zonegroupNames.includes(zonegroup_name)) {
999             const zonegroup = new RgwZonegroup();
1000             zonegroup.name = zonegroup_name;
1001             this.zonegroupList.push(zonegroup);
1002           }
1003           if (zone_name && !this.zoneNames.includes(zone_name)) {
1004             const zone = new RgwZone();
1005             zone.name = zone_name;
1006             this.zoneList.push(zone);
1007           }
1008           if (zonegroup_name === undefined && zone_name === undefined) {
1009             zonegroup_name = 'default';
1010             zone_name = 'default';
1011           }
1012           this.serviceForm.get('realm_name').setValue(realm_name);
1013           this.serviceForm.get('zonegroup_name').setValue(zonegroup_name);
1014           this.serviceForm.get('zone_name').setValue(zone_name);
1015         }
1016         if (this.realmList.length === 0) {
1017           this.showRealmCreationForm = true;
1018         } else {
1019           this.showRealmCreationForm = false;
1020         }
1021       },
1022       (_error) => {
1023         const defaultZone = new RgwZone();
1024         defaultZone.name = 'default';
1025         const defaultZonegroup = new RgwZonegroup();
1026         defaultZonegroup.name = 'default';
1027         this.zoneList.push(defaultZone);
1028         this.zonegroupList.push(defaultZonegroup);
1029       }
1030     );
1031   }
1032
1033   setNvmeServiceId() {
1034     const pool = this.serviceForm.get('pool').value;
1035     const group = this.serviceForm.get('group').value;
1036     if (pool && group) {
1037       this.serviceForm.get('service_id').setValue(`${pool}.${group}`);
1038     } else if (pool) {
1039       this.serviceForm.get('service_id').setValue(pool);
1040     } else if (group) {
1041       this.serviceForm.get('service_id').setValue(group);
1042     } else {
1043       this.serviceForm.get('service_id').setValue(null);
1044     }
1045   }
1046
1047   setNvmeDefaultPool() {
1048     const defaultPool =
1049       this.rbdPools?.find((p: Pool) => p.pool_name === 'rbd')?.pool_name ||
1050       this.rbdPools?.[0].pool_name;
1051     this.serviceForm.get('pool').setValue(defaultPool);
1052   }
1053
1054   requiresServiceId(serviceType: string) {
1055     return ['mds', 'rgw', 'nfs', 'iscsi', 'nvmeof', 'smb', 'ingress'].includes(serviceType);
1056   }
1057
1058   setServiceId(serviceId: string): void {
1059     const requiresServiceId: boolean = this.requiresServiceId(serviceId);
1060     if (requiresServiceId && serviceId === 'nvmeof') {
1061       this.setNvmeDefaultPool();
1062       this.setNvmeServiceId();
1063     } else if (requiresServiceId) {
1064       this.serviceForm.get('service_id').setValue(null);
1065     } else {
1066       this.serviceForm.get('service_id').setValue(serviceId);
1067     }
1068   }
1069
1070   onServiceTypeChange(selectedServiceType: string) {
1071     this.setServiceId(selectedServiceType);
1072
1073     this.serviceIds = this.serviceList
1074       ?.filter((service) => service['service_type'] === selectedServiceType)
1075       .map((service) => service['service_id']);
1076
1077     this.getDefaultPlacementCount(selectedServiceType);
1078
1079     if (selectedServiceType === 'rgw') {
1080       this.setRgwFields();
1081     }
1082     if (selectedServiceType === 'mgmt-gateway') {
1083       let hrefSplitted = window.location.href.split(':');
1084       this.currentURL = hrefSplitted[0] + hrefSplitted[1];
1085       // mgmt-gateway lacks HA for now
1086       this.serviceForm.get('count').disable();
1087     } else {
1088       this.serviceForm.get('count').enable();
1089     }
1090   }
1091
1092   onPlacementChange(selected: string) {
1093     if (selected === 'label') {
1094       this.serviceForm.get('count').setValue(null);
1095     }
1096   }
1097
1098   disableForEditing(serviceType: string) {
1099     const disableForEditKeys = ['service_type', 'service_id'];
1100     disableForEditKeys.forEach((key) => {
1101       this.serviceForm.get(key).disable();
1102     });
1103     switch (serviceType) {
1104       case 'ingress':
1105         this.serviceForm.get('backend_service').disable();
1106         break;
1107       case 'nvmeof':
1108         this.serviceForm.get('pool').disable();
1109         this.serviceForm.get('group').disable();
1110         break;
1111     }
1112   }
1113
1114   searchLabels = (text$: Observable<string>) => {
1115     return merge(
1116       text$.pipe(debounceTime(200), distinctUntilChanged()),
1117       this.labelFocus,
1118       this.labelClick.pipe(filter(() => !this.typeahead.isPopupOpen()))
1119     ).pipe(
1120       map((value) =>
1121         this.labels
1122           .filter((label: string) => label.toLowerCase().indexOf(value.toLowerCase()) > -1)
1123           .slice(0, 10)
1124       )
1125     );
1126   };
1127
1128   fileUpload(files: FileList, controlName: string) {
1129     const file: File = files[0];
1130     const reader = new FileReader();
1131     reader.addEventListener('load', (event: ProgressEvent<FileReader>) => {
1132       const control: AbstractControl = this.serviceForm.get(controlName);
1133       control.setValue(event.target.result);
1134       control.markAsDirty();
1135       control.markAsTouched();
1136       control.updateValueAndValidity();
1137     });
1138     reader.readAsText(file, 'utf8');
1139   }
1140
1141   prePopulateId() {
1142     const control: AbstractControl = this.serviceForm.get('service_id');
1143     const backendService = this.serviceForm.getValue('backend_service');
1144     // Set Id as read-only
1145     control.reset({ value: backendService, disabled: true });
1146   }
1147
1148   onSubmit() {
1149     const self = this;
1150     const values: object = this.serviceForm.getRawValue();
1151     const serviceType: string = values['service_type'];
1152     let taskUrl = `service/${URLVerbs.CREATE}`;
1153     if (this.editing) {
1154       taskUrl = `service/${URLVerbs.EDIT}`;
1155     }
1156     const serviceSpec: object = {
1157       service_type: serviceType,
1158       placement: {},
1159       unmanaged: values['unmanaged']
1160     };
1161     if (serviceType === 'rgw') {
1162       serviceSpec['rgw_realm'] = values['realm_name'] ? values['realm_name'] : null;
1163       serviceSpec['rgw_zonegroup'] =
1164         values['zonegroup_name'] !== 'default' ? values['zonegroup_name'] : null;
1165       serviceSpec['rgw_zone'] = values['zone_name'] !== 'default' ? values['zone_name'] : null;
1166     }
1167
1168     const serviceId: string = values['service_id'];
1169     let serviceName: string = serviceType;
1170     if (_.isString(serviceId) && !_.isEmpty(serviceId) && serviceId !== serviceType) {
1171       serviceName = `${serviceType}.${serviceId}`;
1172       serviceSpec['service_id'] = serviceId;
1173     }
1174
1175     // These services has some fields to be
1176     // filled out even if unmanaged is true
1177     switch (serviceType) {
1178       case 'ingress':
1179         serviceSpec['backend_service'] = values['backend_service'];
1180         serviceSpec['service_id'] = values['backend_service'];
1181         if (_.isNumber(values['frontend_port']) && values['frontend_port'] > 0) {
1182           serviceSpec['frontend_port'] = values['frontend_port'];
1183         }
1184         if (_.isString(values['virtual_ip']) && !_.isEmpty(values['virtual_ip'])) {
1185           serviceSpec['virtual_ip'] = values['virtual_ip'].trim();
1186         }
1187         if (_.isNumber(values['monitor_port']) && values['monitor_port'] > 0) {
1188           serviceSpec['monitor_port'] = values['monitor_port'];
1189         }
1190         break;
1191
1192       case 'nvmeof':
1193         serviceSpec['pool'] = values['pool'];
1194         serviceSpec['group'] = values['group'];
1195         serviceSpec['enable_auth'] = values['enable_mtls'];
1196         if (values['enable_mtls']) {
1197           serviceSpec['root_ca_cert'] = values['root_ca_cert'];
1198           serviceSpec['client_cert'] = values['client_cert'];
1199           serviceSpec['client_key'] = values['client_key'];
1200           serviceSpec['server_cert'] = values['server_cert'];
1201           serviceSpec['server_key'] = values['server_key'];
1202         }
1203         break;
1204       case 'iscsi':
1205         serviceSpec['pool'] = values['pool'];
1206         break;
1207
1208       case 'smb':
1209         serviceSpec['cluster_id'] = values['cluster_id']?.trim();
1210         serviceSpec['config_uri'] = values['config_uri']?.trim();
1211         for (const feature in values['features']) {
1212           if (values['features'][feature]) {
1213             (serviceSpec['features'] = serviceSpec['features'] || []).push(feature);
1214           }
1215         }
1216         serviceSpec['custom_dns'] = values['custom_dns']?.trim();
1217         serviceSpec['join_sources'] = values['join_sources']?.trim();
1218         serviceSpec['user_sources'] = values['user_sources']?.trim();
1219         serviceSpec['include_ceph_users'] = values['include_ceph_users']?.trim();
1220         break;
1221
1222       case 'snmp-gateway':
1223         serviceSpec['credentials'] = {};
1224         serviceSpec['snmp_version'] = values['snmp_version'];
1225         serviceSpec['snmp_destination'] = values['snmp_destination'];
1226         if (values['snmp_version'] === 'V3') {
1227           serviceSpec['engine_id'] = values['engine_id'];
1228           serviceSpec['auth_protocol'] = values['auth_protocol'];
1229           serviceSpec['credentials']['snmp_v3_auth_username'] = values['snmp_v3_auth_username'];
1230           serviceSpec['credentials']['snmp_v3_auth_password'] = values['snmp_v3_auth_password'];
1231           if (values['privacy_protocol'] !== null) {
1232             serviceSpec['privacy_protocol'] = values['privacy_protocol'];
1233             serviceSpec['credentials']['snmp_v3_priv_password'] = values['snmp_v3_priv_password'];
1234           }
1235         } else {
1236           serviceSpec['credentials']['snmp_community'] = values['snmp_community'];
1237         }
1238         break;
1239     }
1240
1241     if (!values['unmanaged']) {
1242       switch (values['placement']) {
1243         case 'hosts':
1244           if (values['hosts'].length > 0) {
1245             serviceSpec['placement']['hosts'] = values['hosts'];
1246           }
1247           break;
1248         case 'label':
1249           serviceSpec['placement']['label'] = values['label'];
1250           break;
1251       }
1252       if (_.isNumber(values['count']) && values['count'] > 0) {
1253         serviceSpec['placement']['count'] = values['count'];
1254       }
1255       switch (serviceType) {
1256         case 'rgw':
1257           if (_.isNumber(values['rgw_frontend_port']) && values['rgw_frontend_port'] > 0) {
1258             serviceSpec['rgw_frontend_port'] = values['rgw_frontend_port'];
1259           }
1260           serviceSpec['ssl'] = values['ssl'];
1261           if (values['ssl']) {
1262             serviceSpec['rgw_frontend_ssl_certificate'] = values['ssl_cert']?.trim();
1263           }
1264           break;
1265         case 'iscsi':
1266           if (_.isString(values['trusted_ip_list']) && !_.isEmpty(values['trusted_ip_list'])) {
1267             serviceSpec['trusted_ip_list'] = values['trusted_ip_list'].trim();
1268           }
1269           if (_.isNumber(values['api_port']) && values['api_port'] > 0) {
1270             serviceSpec['api_port'] = values['api_port'];
1271           }
1272           serviceSpec['api_user'] = values['api_user'];
1273           serviceSpec['api_password'] = values['api_password'];
1274           serviceSpec['api_secure'] = values['ssl'];
1275           if (values['ssl']) {
1276             serviceSpec['ssl_cert'] = values['ssl_cert']?.trim();
1277             serviceSpec['ssl_key'] = values['ssl_key']?.trim();
1278           }
1279           break;
1280         case 'ingress':
1281           serviceSpec['ssl'] = values['ssl'];
1282           if (values['ssl']) {
1283             serviceSpec['ssl_cert'] = values['ssl_cert']?.trim();
1284             serviceSpec['ssl_key'] = values['ssl_key']?.trim();
1285           }
1286           serviceSpec['virtual_interface_networks'] = values['virtual_interface_networks'];
1287           break;
1288         case 'mgmt-gateway':
1289           serviceSpec['ssl_cert'] = values['ssl_cert']?.trim();
1290           serviceSpec['ssl_key'] = values['ssl_key']?.trim();
1291           serviceSpec['enable_auth'] = values['enable_auth'];
1292           serviceSpec['port'] = values['port'];
1293           if (serviceSpec['port'] === (443 || 80)) {
1294             // omit port default values due to issues with redirect_url on the backend
1295             delete serviceSpec['port'];
1296           }
1297           serviceSpec['ssl_protocols'] = [];
1298           if (values['ssl_protocols'] != this.DEFAULT_SSL_PROTOCOL_ITEM) {
1299             for (const key of Object.keys(values['ssl_protocols'])) {
1300               serviceSpec['ssl_protocols'].push(values['ssl_protocols'][key]['content']);
1301             }
1302           }
1303           serviceSpec['ssl_ciphers'] = values['ssl_ciphers']?.trim().split(':');
1304           break;
1305         case 'grafana':
1306           serviceSpec['port'] = values['grafana_port'];
1307           serviceSpec['initial_admin_password'] = values['grafana_admin_password'];
1308           break;
1309         case 'oauth2-proxy':
1310           serviceSpec['provider_display_name'] = values['provider_display_name']?.trim();
1311           serviceSpec['client_id'] = values['client_id']?.trim();
1312           serviceSpec['client_secret'] = values['client_secret']?.trim();
1313           serviceSpec['oidc_issuer_url'] = values['oidc_issuer_url']?.trim();
1314           serviceSpec['https_address'] = values['https_address']?.trim();
1315           serviceSpec['redirect_url'] = values['redirect_url']?.trim();
1316           serviceSpec['allowlist_domains'] = values['allowlist_domains']
1317             .split(',')
1318             .map((domain: string) => {
1319               return domain.trim();
1320             });
1321           if (values['ssl']) {
1322             serviceSpec['ssl_cert'] = values['ssl_cert']?.trim();
1323             serviceSpec['ssl_key'] = values['ssl_key']?.trim();
1324           }
1325           break;
1326       }
1327     }
1328     this.taskWrapperService
1329       .wrapTaskAroundCall({
1330         task: new FinishedTask(taskUrl, {
1331           service_name: serviceName
1332         }),
1333         call: this.editing
1334           ? this.cephServiceService.update(serviceSpec)
1335           : this.cephServiceService.create(serviceSpec)
1336       })
1337       .subscribe({
1338         error() {
1339           self.serviceForm.setErrors({ cdSubmitButton: true });
1340         },
1341         complete: () => {
1342           this.pageURL === 'services'
1343             ? this.router.navigate([this.pageURL, { outlets: { modal: null } }])
1344             : this.activeModal.close();
1345         }
1346       });
1347   }
1348
1349   clearValidations() {
1350     const snmpVersion = this.serviceForm.getValue('snmp_version');
1351     const privacyProtocol = this.serviceForm.getValue('privacy_protocol');
1352     if (snmpVersion === 'V3') {
1353       this.serviceForm.get('snmp_community').clearValidators();
1354     } else {
1355       this.serviceForm.get('engine_id').clearValidators();
1356       this.serviceForm.get('auth_protocol').clearValidators();
1357       this.serviceForm.get('privacy_protocol').clearValidators();
1358       this.serviceForm.get('snmp_v3_auth_username').clearValidators();
1359       this.serviceForm.get('snmp_v3_auth_password').clearValidators();
1360     }
1361     if (privacyProtocol === null) {
1362       this.serviceForm.get('snmp_v3_priv_password').clearValidators();
1363     }
1364   }
1365
1366   createMultisiteSetup() {
1367     this.bsModalRef = this.modalService.show(CreateRgwServiceEntitiesComponent, {
1368       size: 'lg'
1369     });
1370     this.bsModalRef.componentInstance.submitAction.subscribe(() => {
1371       this.setRgwFields();
1372     });
1373   }
1374 }