:Description: Number of OSDs requested to send data during recovery of
a single chunk. *d* needs to be chosen such that
- k+1 <= d <= k+m-1. Larger the *d*, the better the savings.
+ k+1 <= d <= k+m-1. The larger the *d*, the better the savings.
:Type: Integer
:Required: No.
"""
config = mgr.get('config')
return {
- # Because 'shec' is experimental it's not included
- 'plugins': config['osd_erasure_code_plugins'].split() + ['shec'],
+ # Because 'shec' and 'clay' are experimental they're not included
+ 'plugins': config['osd_erasure_code_plugins'].split() + ['shec', 'clay'],
'directory': config['erasure_code_dir'],
'nodes': mgr.get('osd_map_tree')['nodes'],
'names': [name for name, _ in
</div>
</div>
+ <div class="form-group row"
+ *ngIf="plugin === 'clay'">
+ <label for="d"
+ class="cd-col-form-label">
+ <span class="required"
+ i18n>Helper chunks (d)</span>
+ <cd-helper [html]="tooltips.plugins.clay.d">
+ </cd-helper>
+ </label>
+ <div class="cd-col-form-input">
+ <div class="input-group">
+ <input type="number"
+ id="d"
+ name="d"
+ class="form-control"
+ placeholder="Helper chunks..."
+ formControlName="d">
+ <span class="input-group-append">
+ <button class="btn btn-light"
+ id="d-calc-btn"
+ ngbTooltip="Set d manually or use the plugin's default calculation that maximizes d."
+ i18n-ngbTooltip
+ type="button"
+ (click)="toggleDCalc()">
+ <i [ngClass]="dCalc ? icons.unlock : icons.lock"
+ aria-hidden="true"></i>
+ </button>
+ </span>
+ </div>
+ <span class="form-text text-muted"
+ *ngIf="dCalc"
+ i18n>D is automatically updated on k and m changes</span>
+ <ng-container
+ *ngIf="!dCalc">
+ <span class="form-text text-muted"
+ *ngIf="getDMin() < getDMax()"
+ i18n>D can be set from {{getDMin()}} to {{getDMax()}}</span>
+ <span class="form-text text-muted"
+ *ngIf="getDMin() === getDMax()"
+ i18n>D can only be set to {{getDMax()}}</span>
+ </ng-container>
+ <span class="invalid-feedback"
+ *ngIf="form.showError('d', frm, 'dMin')"
+ i18n>D has to be greater than k ({{getDMin()}}).</span>
+ <span class="invalid-feedback"
+ *ngIf="form.showError('d', frm, 'dMax')"
+ i18n>D has to be lower than k + m ({{getDMax()}}).</span>
+ </div>
+ </div>
+
<div class="form-group row"
*ngIf="plugin === PLUGIN.LRC">
<label class="cd-col-form-label"
</div>
<div class="form-group row"
- *ngIf="[PLUGIN.JERASURE, PLUGIN.ISA].includes(plugin)">
+ *ngIf="PLUGIN.CLAY === plugin">
+ <label for="scalar_mds"
+ class="cd-col-form-label">
+ <ng-container i18n>Scalar mds</ng-container>
+ <cd-helper [html]="tooltips.plugins.clay.scalar_mds">
+ </cd-helper>
+ </label>
+ <div class="cd-col-form-input">
+ <select class="form-control custom-select"
+ id="scalar_mds"
+ name="scalar_mds"
+ formControlName="scalar_mds">
+ <option *ngFor="let plugin of [PLUGIN.JERASURE, PLUGIN.ISA, PLUGIN.SHEC]"
+ [ngValue]="plugin">
+ {{ plugin }}
+ </option>
+ </select>
+ </div>
+ </div>
+
+ <div class="form-group row"
+ *ngIf="[PLUGIN.JERASURE, PLUGIN.ISA, PLUGIN.CLAY].includes(plugin)">
<label for="technique"
class="cd-col-form-label">
<ng-container i18n>Technique</ng-container>
import { HttpClientTestingModule } from '@angular/common/http/testing';
import { ComponentFixture, TestBed } from '@angular/core/testing';
-import { By } from '@angular/platform-browser';
import { RouterTestingModule } from '@angular/router/testing';
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
Mocks
} from '../../../../testing/unit-test-helper';
import { ErasureCodeProfileService } from '../../../shared/api/erasure-code-profile.service';
+import { CrushNode } from '../../../shared/models/crush-node';
import { ErasureCodeProfile } from '../../../shared/models/erasure-code-profile';
import { TaskWrapperService } from '../../../shared/services/task-wrapper.service';
import { PoolModule } from '../pool.module';
let fixture: ComponentFixture<ErasureCodeProfileFormModalComponent>;
let formHelper: FormHelper;
let fixtureHelper: FixtureHelper;
- let data: {};
+ let data: { plugins: string[]; names: string[]; nodes: CrushNode[] };
+
+ const expectTechnique = (current: string) =>
+ expect(component.form.getValue('technique')).toBe(current);
+
+ const expectTechniques = (techniques: string[], current: string) => {
+ expect(component.techniques).toEqual(techniques);
+ expectTechnique(current);
+ };
+
+ const expectRequiredControls = (controlNames: string[]) => {
+ controlNames.forEach((name) => {
+ const value = component.form.getValue(name);
+ formHelper.expectValid(name);
+ formHelper.expectErrorChange(name, null, 'required');
+ // This way other fields won't fail through getting invalid.
+ formHelper.expectValidChange(name, value);
+ });
+ fixtureHelper.expectIdElementsVisible(controlNames, true);
+ };
configureTestBed({
imports: [
showDefaults('isa');
});
+ it('should change technique to default if not available in other plugin', () => {
+ expectTechnique('reed_sol_van');
+ formHelper.setValue('technique', 'blaum_roth');
+ expectTechnique('blaum_roth');
+ formHelper.setValue('plugin', 'isa');
+ expectTechnique('reed_sol_van');
+ formHelper.setValue('plugin', 'clay');
+ formHelper.expectValidChange('scalar_mds', 'shec');
+ expectTechnique('single');
+ });
+
describe(`for 'jerasure' plugin (default)`, () => {
it(`requires 'm' and 'k'`, () => {
- formHelper.expectErrorChange('k', null, 'required');
- formHelper.expectErrorChange('m', null, 'required');
+ expectRequiredControls(['k', 'm']);
});
it(`should show 'packetSize' and 'technique'`, () => {
fixtureHelper.expectIdElementsVisible(['packetSize', 'technique'], true);
});
+ it('should show available techniques', () => {
+ expectTechniques(
+ [
+ 'reed_sol_van',
+ 'reed_sol_r6_op',
+ 'cauchy_orig',
+ 'cauchy_good',
+ 'liberation',
+ 'blaum_roth',
+ 'liber8tion'
+ ],
+ 'reed_sol_van'
+ );
+ });
+
it(`should not show any other plugin specific form control`, () => {
- fixtureHelper.expectIdElementsVisible(['c', 'l', 'crushLocality'], false);
+ fixtureHelper.expectIdElementsVisible(
+ ['c', 'l', 'crushLocality', 'd', 'scalar_mds'],
+ false
+ );
});
it('should not allow "k" to be changed more than possible', () => {
});
it(`does require 'm' and 'k'`, () => {
- formHelper.expectErrorChange('k', null, 'required');
- formHelper.expectErrorChange('m', null, 'required');
+ expectRequiredControls(['k', 'm']);
});
it(`should show 'technique'`, () => {
fixtureHelper.expectIdElementsVisible(['technique'], true);
- expect(fixture.debugElement.query(By.css('#technique'))).toBeTruthy();
+ });
+
+ it('should show available techniques', () => {
+ expectTechniques(['reed_sol_van', 'cauchy'], 'reed_sol_van');
});
it(`should not show any other plugin specific form control`, () => {
- fixtureHelper.expectIdElementsVisible(['c', 'l', 'crushLocality', 'packetSize'], false);
+ fixtureHelper.expectIdElementsVisible(
+ ['c', 'l', 'crushLocality', 'packetSize', 'd', 'scalar_mds'],
+ false
+ );
});
it('should not allow "k" to be changed more than possible', () => {
});
it(`requires 'm', 'l' and 'k'`, () => {
- formHelper.expectErrorChange('k', null, 'required');
- formHelper.expectErrorChange('m', null, 'required');
- formHelper.expectErrorChange('l', null, 'required');
+ expectRequiredControls(['k', 'm', 'l']);
});
it(`should show 'l' and 'crushLocality'`, () => {
});
it(`should not show any other plugin specific form control`, () => {
- fixtureHelper.expectIdElementsVisible(['c', 'packetSize', 'technique'], false);
+ fixtureHelper.expectIdElementsVisible(
+ ['c', 'packetSize', 'technique', 'd', 'scalar_mds'],
+ false
+ );
});
it('should not allow "k" to be changed more than possible', () => {
});
it(`does require 'm', 'c' and 'k'`, () => {
- formHelper.expectErrorChange('k', null, 'required');
- formHelper.expectErrorChange('m', null, 'required');
- formHelper.expectErrorChange('c', null, 'required');
- });
-
- it(`should show 'c'`, () => {
- fixtureHelper.expectIdElementsVisible(['c'], true);
+ expectRequiredControls(['k', 'm', 'c']);
});
it(`should not show any other plugin specific form control`, () => {
fixtureHelper.expectIdElementsVisible(
- ['l', 'crushLocality', 'packetSize', 'technique'],
+ ['l', 'crushLocality', 'packetSize', 'technique', 'd', 'scalar_mds'],
false
);
});
formHelper.expectValid('k');
});
});
+
+ describe(`for 'clay' plugin`, () => {
+ beforeEach(() => {
+ formHelper.setValue('plugin', 'clay');
+ // Through this change d has a valid range from 4 to 7
+ formHelper.expectValidChange('k', 3);
+ formHelper.expectValidChange('m', 5);
+ });
+
+ it(`does require 'm', 'c', 'd', 'scalar_mds' and 'k'`, () => {
+ fixtureHelper.clickElement('#d-calc-btn');
+ expectRequiredControls(['k', 'm', 'd', 'scalar_mds']);
+ });
+
+ it(`should not show any other plugin specific form control`, () => {
+ fixtureHelper.expectIdElementsVisible(['l', 'crushLocality', 'packetSize', 'c'], false);
+ });
+
+ it('should show default values for d and scalar_mds', () => {
+ expect(component.form.getValue('d')).toBe(7); // (k+m-1)
+ expect(component.form.getValue('scalar_mds')).toBe('jerasure');
+ });
+
+ it('should auto change d if auto calculation is enabled (default)', () => {
+ formHelper.expectValidChange('k', 4);
+ expect(component.form.getValue('d')).toBe(8);
+ });
+
+ it('should have specific techniques for scalar_mds jerasure', () => {
+ expectTechniques(
+ ['reed_sol_van', 'reed_sol_r6_op', 'cauchy_orig', 'cauchy_good', 'liber8tion'],
+ 'reed_sol_van'
+ );
+ });
+
+ it('should have specific techniques for scalar_mds isa', () => {
+ formHelper.expectValidChange('scalar_mds', 'isa');
+ expectTechniques(['reed_sol_van', 'cauchy'], 'reed_sol_van');
+ });
+
+ it('should have specific techniques for scalar_mds shec', () => {
+ formHelper.expectValidChange('scalar_mds', 'shec');
+ expectTechniques(['single', 'multiple'], 'single');
+ });
+
+ describe('Validity of d', () => {
+ beforeEach(() => {
+ // Don't automatically change d - the only way to get d invalid
+ fixtureHelper.clickElement('#d-calc-btn');
+ });
+
+ it('should not automatically change d if k or m have been changed', () => {
+ formHelper.expectValidChange('m', 4);
+ formHelper.expectValidChange('k', 5);
+ expect(component.form.getValue('d')).toBe(7);
+ });
+
+ it('should trigger dMin through change of d', () => {
+ formHelper.expectErrorChange('d', 3, 'dMin');
+ });
+
+ it('should trigger dMax through change of d', () => {
+ formHelper.expectErrorChange('d', 8, 'dMax');
+ });
+
+ it('should trigger dMin through change of k and m', () => {
+ formHelper.expectValidChange('m', 2);
+ formHelper.expectValidChange('k', 7);
+ formHelper.expectError('d', 'dMin');
+ });
+
+ it('should trigger dMax through change of m', () => {
+ formHelper.expectValidChange('m', 3);
+ formHelper.expectError('d', 'dMax');
+ });
+
+ it('should remove dMax through change of k', () => {
+ formHelper.expectValidChange('m', 3);
+ formHelper.expectError('d', 'dMax');
+ formHelper.expectValidChange('k', 5);
+ formHelper.expectValid('d');
+ });
+ });
+ });
});
describe('submission', () => {
testCreation();
});
});
+
+ describe(`'clay' usage`, () => {
+ beforeEach(() => {
+ ecpChange('name', 'clayProfile');
+ ecpChange('plugin', 'clay');
+ // Setting expectations
+ submittedEcp.k = 4;
+ submittedEcp.m = 2;
+ submittedEcp.d = 5;
+ submittedEcp.scalar_mds = 'jerasure';
+ delete submittedEcp.packetsize;
+ });
+
+ it('should be able to create a profile with only plugin and name', () => {
+ formHelper.setMultipleValues(ecp, true);
+ testCreation();
+ });
+
+ it('should send profile with a changed d', () => {
+ formHelper.setMultipleValues(ecp, true);
+ ecpChange('d', '5');
+ submittedEcp.d = 5;
+ testCreation();
+ });
+
+ it('should send profile with a changed k which automatically changes d', () => {
+ ecpChange('k', 5);
+ formHelper.setMultipleValues(ecp, true);
+ submittedEcp.d = 6;
+ testCreation();
+ });
+
+ it('should send profile with a changed sclara_mds', () => {
+ ecpChange('scalar_mds', 'shec');
+ formHelper.setMultipleValues(ecp, true);
+ submittedEcp.scalar_mds = 'shec';
+ submittedEcp.technique = 'single';
+ testCreation();
+ });
+
+ it('should not send the profile with unsupported fields', () => {
+ formHelper.setMultipleValues(ecp, true);
+ formHelper.setValue('l', 8, true);
+ testCreation();
+ });
+ });
});
});
import { ErasureCodeProfileService } from '../../../shared/api/erasure-code-profile.service';
import { CrushNodeSelectionClass } from '../../../shared/classes/crush.node.selection.class';
import { ActionLabelsI18n } from '../../../shared/constants/app.constants';
+import { Icons } from '../../../shared/enum/icons.enum';
import { CdFormBuilder } from '../../../shared/forms/cd-form-builder';
import { CdFormGroup } from '../../../shared/forms/cd-form-group';
import { CdValidators } from '../../../shared/forms/cd-validators';
PLUGIN = {
LRC: 'lrc', // Locally Repairable Erasure Code
SHEC: 'shec', // Shingled Erasure Code
+ CLAY: 'clay', // Coupled LAYer
JERASURE: 'jerasure', // default
ISA: 'isa' // Intel Storage Acceleration
};
plugin = this.PLUGIN.JERASURE;
+ icons = Icons;
form: CdFormGroup;
plugins: string[];
techniques: string[];
action: string;
resource: string;
+ dCalc: boolean;
lrcGroups: number;
lrcMultiK: number;
crushRoot: null, // Will be preselected
crushDeviceClass: '', // Will be preselected
directory: '',
- // Only for 'jerasure' and 'isa' use
+ // Only for 'jerasure', 'clay' and 'isa' use
technique: 'reed_sol_van',
// Only for 'jerasure' use
packetSize: [2048, [Validators.min(1)]],
Validators.min(1),
CdValidators.custom('cGreaterM', (v: number) => this.shecDurabilityValidation(v))
]
- ]
+ ],
+ // Only for 'clay' use
+ d: [
+ 5, // Will be overwritten with plugin defaults (k+m-1) = k+1 <= d <= k+m-1
+ [
+ Validators.required,
+ CdValidators.custom('dMin', (v: number) => this.dMinValidation(v)),
+ CdValidators.custom('dMax', (v: number) => this.dMaxValidation(v))
+ ]
+ ],
+ scalar_mds: [this.PLUGIN.JERASURE, [Validators.required]] // jerasure or isa or shec
});
- this.form.get('k').valueChanges.subscribe(() => this.updateValidityOnChange(['m', 'l']));
- this.form.get('m').valueChanges.subscribe(() => this.updateValidityOnChange(['k', 'l', 'c']));
+ this.toggleDCalc();
+ this.form.get('k').valueChanges.subscribe(() => this.updateValidityOnChange(['m', 'l', 'd']));
+ this.form
+ .get('m')
+ .valueChanges.subscribe(() => this.updateValidityOnChange(['k', 'l', 'c', 'd']));
this.form.get('l').valueChanges.subscribe(() => this.updateValidityOnChange(['k', 'm']));
this.form.get('plugin').valueChanges.subscribe((plugin) => this.onPluginChange(plugin));
+ this.form.get('scalar_mds').valueChanges.subscribe(() => this.setClayDefaultsForScalar());
}
private baseValueValidation(dataChunk: boolean = false): boolean {
}, 'shec');
}
+ private dMinValidation(d: number): boolean {
+ return this.validValidation(() => this.getDMin() > d, 'clay');
+ }
+
+ getDMin(): number {
+ return this.form.getValue('k') + 1;
+ }
+
+ private dMaxValidation(d: number): boolean {
+ return this.validValidation(() => d > this.getDMax(), 'clay');
+ }
+
+ getDMax(): number {
+ const m = this.form.getValue('m');
+ const k = this.form.getValue('k');
+ return k + m - 1;
+ }
+
+ toggleDCalc() {
+ this.dCalc = !this.dCalc;
+ this.form.get('d')[this.dCalc ? 'disable' : 'enable']();
+ this.calculateD();
+ }
+
+ private calculateD() {
+ if (this.plugin !== this.PLUGIN.CLAY || !this.dCalc) {
+ return;
+ }
+ this.form.silentSet('d', this.getDMax());
+ }
+
private updateValidityOnChange(names: string[]) {
- names.forEach((name) => this.form.get(name).updateValueAndValidity({ emitEvent: false }));
+ names.forEach((name) => {
+ if (name === 'd') {
+ this.calculateD();
+ }
+ this.form.get(name).updateValueAndValidity({ emitEvent: false });
+ });
}
private onPluginChange(plugin: string) {
this.setIsaDefaults();
} else if (plugin === this.PLUGIN.SHEC) {
this.setShecDefaults();
+ } else if (plugin === this.PLUGIN.CLAY) {
+ this.setClayDefaults();
}
- this.updateValidityOnChange(['m']); // Triggers k, m, c and l
+ this.updateValidityOnChange(['m']); // Triggers k, m, c, d and l
}
private setJerasureDefaults() {
- this.setDefaults({
- k: 4,
- m: 2
- });
this.techniques = [
'reed_sol_van',
'reed_sol_r6_op',
'blaum_roth',
'liber8tion'
];
+ this.setDefaults({
+ k: 4,
+ m: 2,
+ technique: 'reed_sol_van'
+ });
}
private setLrcDefaults() {
* if they are not set, therefore it's fine to mark them as required in order to get
* strange values that weren't set.
*/
+ this.techniques = ['reed_sol_van', 'cauchy'];
this.setDefaults({
k: 7,
- m: 3
+ m: 3,
+ technique: 'reed_sol_van'
});
- this.techniques = ['reed_sol_van', 'cauchy'];
}
private setShecDefaults() {
});
}
+ private setClayDefaults() {
+ /**
+ * Actually d and scalar_mds are not required - but they will be set to show the default values
+ * in case if they are not set, therefore it's fine to mark them as required in order to not get
+ * strange values that weren't set.
+ *
+ * As d would be set to the value k+m-1 for the greatest savings, the form will
+ * automatically update d if the automatic calculation is activated (default).
+ */
+ this.setDefaults({
+ k: 4,
+ m: 2,
+ // d: 5, <- Will be automatically update to 5
+ scalar_mds: this.PLUGIN.JERASURE
+ });
+ this.setClayDefaultsForScalar();
+ }
+
+ private setClayDefaultsForScalar() {
+ const plugin = this.form.getValue('scalar_mds');
+ let defaultTechnique = 'reed_sol_van';
+ if (plugin === this.PLUGIN.JERASURE) {
+ this.techniques = [
+ 'reed_sol_van',
+ 'reed_sol_r6_op',
+ 'cauchy_orig',
+ 'cauchy_good',
+ 'liber8tion'
+ ];
+ } else if (plugin === this.PLUGIN.ISA) {
+ this.techniques = ['reed_sol_van', 'cauchy'];
+ } else {
+ // this.PLUGIN.SHEC
+ defaultTechnique = 'single';
+ this.techniques = ['single', 'multiple'];
+ }
+ this.setDefaults({ technique: defaultTechnique });
+ }
+
private setDefaults(defaults: object) {
Object.keys(defaults).forEach((controlName) => {
const control = this.form.get(controlName);
const value = control.value;
- let overwrite = control.pristine;
/**
* As k, m, c and l are now set touched and dirty on the beginning, plugin change will
* overwrite their values as we can't determine if the user has changed anything.
* k and m can have two default values where as l and c can only have one,
* so there is no need to overwrite them.
*/
- if ('k' === controlName) {
- overwrite = [4, 7].includes(value);
- } else if ('m' === controlName) {
- overwrite = [2, 3].includes(value);
- }
+ const overwrite =
+ control.pristine ||
+ (controlName === 'technique' && !this.techniques.includes(value)) ||
+ (controlName === 'k' && [4, 7].includes(value)) ||
+ (controlName === 'm' && [2, 3].includes(value));
if (overwrite) {
- this.form.get(controlName).setValue(defaults[controlName]);
+ control.setValue(defaults[controlName]); // also validates new value
+ } else {
+ control.updateValueAndValidity();
}
});
}
* fields got changed before by the user.
*/
private preValidateNumericInputFields() {
- const kml = ['k', 'm', 'l', 'c'].map((name) => this.form.get(name));
+ const kml = ['k', 'm', 'l', 'c', 'd'].map((name) => this.form.get(name));
kml.forEach((control) => {
control.markAsTouched();
control.markAsDirty();
});
- kml[1].updateValueAndValidity(); // Update validity of k, m, c and l
+ kml[1].updateValueAndValidity(); // Update validity of k, m, c, d and l
}
onSubmit() {
private createJson() {
const pluginControls = {
- technique: [this.PLUGIN.ISA, this.PLUGIN.JERASURE],
+ technique: [this.PLUGIN.ISA, this.PLUGIN.JERASURE, this.PLUGIN.CLAY],
packetSize: [this.PLUGIN.JERASURE],
l: [this.PLUGIN.LRC],
crushLocality: [this.PLUGIN.LRC],
- c: [this.PLUGIN.SHEC]
+ c: [this.PLUGIN.SHEC],
+ d: [this.PLUGIN.CLAY],
+ scalar_mds: [this.PLUGIN.CLAY]
};
const ecp = new ErasureCodeProfile();
const plugin = this.form.getValue('plugin');
carefully. All of reed_sol_r6_op, liberation, blaum_roth, liber8tion are RAID6 equivalents
in the sense that they can only be configured with m=2.`,
packetSize: $localize`The encoding will be done on packets of bytes size at a time.
- Chosing the right packet size is difficult.
+ Choosing the right packet size is difficult.
The jerasure documentation contains extensive information on this topic.`
},
lrc: {
c: $localize`The number of parity chunks each of which includes each data chunk in its
calculation range. The number is used as a durability estimator. For instance, if c=2,
2 OSDs can be down without losing data.`
+ },
+ clay: {
+ description: $localize`CLAY (short for coupled-layer) codes are erasure codes designed to
+ bring about significant savings in terms of network bandwidth and disk IO when a failed
+ node/OSD/rack is being repaired.`,
+ d: $localize`Number of OSDs requested to send data during recovery of a single chunk.
+ d needs to be chosen such that k+1 <= d <= k+m-1. The larger the d, the better
+ the savings.`,
+ scalar_mds: $localize`scalar_mds specifies the plugin that is used as a building block
+ in the layered construction. It can be one of jerasure, isa, shec.`,
+ technique: $localize`technique specifies the technique that will be picked
+ within the 'scalar_mds' plugin specified. Supported techniques
+ are 'reed_sol_van', 'reed_sol_r6_op', 'cauchy_orig',
+ 'cauchy_good', 'liber8tion' for jerasure, 'reed_sol_van',
+ 'cauchy' for isa and 'single', 'multiple' for shec.`
}
},
m?: number;
c?: number;
l?: number;
+ d?: number;
packetsize?: number;
technique?: string;
+ scalar_mds?: 'jerasure' | 'isa' | 'shec';
'crush-root'?: string;
'crush-locality'?: string;
'crush-failure-domain'?: string;
projects / ceph.git / commitdiff