public string[] LayerMasks {get; set;}

//Get the layer to be masked
var lineLyrToBeMasked = MapView.Active.Map.Layers.FirstOrDefault(lyr => lyr.Name == "TestLine") as FeatureLayer;
//Get the layer's definition
var lyrDefn = lineLyrToBeMasked.GetDefinition();
//Create an array of Masking layers (polygon only)
//Set the LayerMasks property of the Masked layer
lyrDefn.LayerMasks = new string[] { "CIMPATH=map3/testpoly.xml" };
//Re-set the Masked layer's defintion
lineLyrToBeMasked.SetDefinition(lyrDefn);