JMP > JSL > Script Acceleration for Multiple Call to Fit Model (N > 2000)?

Thierry_S

Hi JMP Community,

The size of my datasets is getting bigger and the basic scripts I created a while back (see below) are not keeping up from a speed point of view.

I would greatly appreciate hearing about ideas on how to optimize this type of script for better speed. For example, I suspect that operations like "Concatenate" are relatively slow, and it might make sense to work with matrices for this type of operation.

Names Default to Here (1);

dt = Current Data Table ();

CList = dt << Get Column Names (string);

dtoutl = New Table ("LSM OUTPUT", "invisible");
dtoutp = New Table ("PVal OUTPUT", "invisible");


For (i = 1, i<= 2000, i++,  

	fm = dt <<	Fit Model(
					Y( Column (i) ),
					Effects( :VISIT, :TREATMENT, :VISIT * :TREATMENT, Column (i+2000) ),
					Personality( "Standard Least Squares" ),
					Emphasis( "Minimal Report" ),
					Run(
						Column (i) << {Summary of Fit( 1 ), Analysis of Variance( 1 ),
						Parameter Estimates( 1 ), Scaled Estimates( 0 ),
						Plot Actual by Predicted( 0 ), Plot Regression( 0 ),
						Plot Residual by Predicted( 0 ), Plot Studentized Residuals( 0 ),
						Plot Effect Leverage( 0 ), Plot Residual by Normal Quantiles( 0 ),
						Box Cox Y Transformation( 0 ), {:VISIT * :TRT01A <<
						{LSMeans Contrast( [0 0 1 -1 0 0, 0 0 0 0 1 -1] )}}}
					)
					
				);
	fmr = report (fm);
	
	table_list = fmr << xpath ("//OutlineBox [text() = 'VISIT*TREATMENT']//TableBox");
	
	temp_lsm = table_list [1];
	temp_pval = table_list [2];
	
	temp_tbl_lsm = temp_lsm << Make into Data Table (invisible (1));
	temp_tbl_pval = temp_pval << Make into Data Table (invisible (1));
	
	temp_tbl_lsm << New column ("ID", character, nominal, << set each value (CList [i]));
	temp_tbl_pval << New column ("ID", character, nominal, << set each value (CList [i]));
	
	fmr << close window;
	
	dtoutl << Concatenate (Data table (temp_tbl_lsm), "Append to First Table");
	Wait (0);
	dtoutp << Concatenate (Data table (temp_tbl_pval), "Append to First Table");
	
	Close (temp_tbl_lsm, NoSave);	
	Close (temp_tbl_pval, NoSave);	
		
	
);


dtoutl << Bring Window to Front;
dtoutp << Bring Window to Front;

All feedback is welcome.

Best,

TS

Thierry_S

Hi JMP Community,

 

I have experimented with matrices as receptacles for aggregating long collections of analysis output tables, and so far I have accelerated my execution time by ~5 folds (see below). The only minor inconvenience is that I have had to rebuild the annotations for the captured output tables because (obviously) matrices do not hold text.

Names Default to Here (1);

st = Today();

dt = Current Data Table ();
dts = dt << Summary( Group( :VISIT, :TRT01A ), Freq( "None" ), Weight( "None" ), invisible );
annot_lsm = {};
met_l = {};
annot_pv = {};

CList = dt << Get Column Names (string);

For (j = 1, j <= N Rows (dts), j++,
	insert into (annot_lsm, dts:VISIT [j] || ", " || dts:TRT01A [j]);
);

annot_pv = annot_lsm;

annot_pv = Concat To(annot_pv, {"Estimate","Std Error", "t Ratio", "Prob>|t|", "SS", "Lower 95%", "Upper 95%"});

Close (dts, NoSave);

mtx_lsm = [];
mtx_pval = [];

For (i = 1, i<= 2000, i++,  //2000

	fm = dt <<	Fit Model(
					Y( Column (i) ),
					Effects( :VISIT, :TRT01A, :VISIT * :TRT01A, Column (i+2000) ),
					Personality( "Standard Least Squares" ),
					Emphasis( "Minimal Report" ),
					Run(
						Column (i) << {Summary of Fit( 1 ), Analysis of Variance( 1 ),
						Parameter Estimates( 1 ), Scaled Estimates( 0 ),
						Plot Actual by Predicted( 0 ), Plot Regression( 0 ),
						Plot Residual by Predicted( 0 ), Plot Studentized Residuals( 0 ),
						Plot Effect Leverage( 0 ), Plot Residual by Normal Quantiles( 0 ),
						Box Cox Y Transformation( 0 ), {:VISIT * :TRT01A <<
						{LSMeans Contrast( [0 0 1 -1 0 0, 0 0 0 0 1 -1] )}}}
					)
					
				);
	
	fmr = report (fm);
	
	insert into (met_l, CList [i]);
	
	table_list = fmr << xpath ("//OutlineBox [text() = 'VISIT*TRT01A']//TableBox");
	
	temp_lsm = table_list [1];
	temp_pval = table_list [2];
	
	temp_mtx_lsm = temp_lsm << Get as Matrix;
	temp_mtx_pval = temp_pval << Get as Matrix;
	
	fmr << close window;
	
	mtx_lsm = mtx_lsm |/ temp_mtx_lsm;
	mtx_pval = mtx_pval |/ temp_mtx_pval ;
		
);
dtoutl = as Table (mtx_lsm);
dtoutp = as Table (mtx_pval);

dtoutl << new column ("COMP", Character, Nominal);
dtoutl << new column ("MET_ID", Character, Nominal);
dtoutl:Col1 << set name ("LSMEAN");
dtoutl:Col2 << set name ("STDERR");
dtoutl:Col3 << set name ("UPPER CI");
dtoutl:Col4 << set name ("LOWER CI");

dtoutl << Begin Data Update;
	For Each Row( dtoutl, :COMP = annot_lsm[Sequence( 1, N Items (annot_lsm), 1, 1 )] );
	For each row( dtoutl, :MET_ID = met_l[Sequence (1,N Items (met_l), 1, 6)] );
dtoutl << End Data Update;

dtoutp << new column ("LABEL", Character, Nominal);
dtoutp << new column ("MET_ID", Character, Nominal);

dtoutp << Begin Data Update;
	For Each Row( dtoutp, :LABEL = annot_pv[Sequence( 1, N Items (annot_pv), 1, 1 )] );
	For each row( dtoutp, :MET_ID = met_l[Sequence (1,N items (met_l), 1, 13)] );
dtoutp << End Data Update;