単回帰モデルの最尤推定

単回帰モデルの最尤推定

単回帰モデル

$$\begin{array}{rcl}{y}_i& =& \alpha +\beta x_i+{ϵ}_i(i=1,\cdots ,n)\\ & & {ϵ}_i\stackrel{iid}{\sim }N(0,{\sigma }^2)\cdots \mathrm{独}\mathrm{立}\mathrm{同}\mathrm{一}\mathrm{分}\mathrm{布}(independentandidenticallydistributed;IID,i.i.d.,iid)\end{array}$$

対数尤度凾数

対数尤度凾数は以下のようになる． $$\begin{array}{rcl}f(y_1,\cdots ,y_n;\alpha ,\beta ,{\sigma }^2)& =& \prod _{i=1}^n\frac{1}{\sqrt{2\pi {\sigma }^2}}{e}^{-\frac{1}{2{\sigma }^2}{(y_i-\alpha -\beta x_i)}^2}\\ & =& {\left(2\pi \right)}^{-\frac{n}{2}}{\left({\sigma }^2\right)}^{-\frac{n}{2}}{e}^{-\frac{1}{2{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2}\\ l(\alpha ,\beta ,{\sigma }^2;y_1,\cdots ,y_n)& =& \log \left\{{\left(2\pi \right)}^{-\frac{n}{2}}{\left({\sigma }^2\right)}^{-\frac{n}{2}}{e}^{-\frac{1}{2{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2}\right\}\\ & =& \log \left\{{\left(2\pi \right)}^{-\frac{n}{2}}\right\}+\log \left\{{\left({\sigma }^2\right)}^{-\frac{n}{2}}\right\}+\log \left\{e^{-\frac{1}{2{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2}\right\}\\ & =& -\frac{n}{2}\log \left(2\pi \right)-\frac{n}{2}\log \left({\sigma }^2\right)-\frac{1}{2{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\end{array}$$

スコア凾数

スコア凾数は以下のようになる． $$\begin{array}{rcl}\frac{\partial l}{\partial \alpha }& =& \frac{\partial l}{\partial \alpha }\{-\frac{1}{2{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\}\\ & =& -\frac{1}{2{\sigma }^2}\sum _{i=1}^n\frac{\partial l}{\partial \alpha }{(y_i-\alpha -\beta x_i)}^2\\ & =& -\frac{1}{2{\sigma }^2}\sum _{i=1}^n(y_i-\alpha -\beta x_i)(-1)\\ & =& -\frac{-1}{2{\sigma }^2}\sum _{i=1}^n(y_i-\alpha -\beta x_i)\\ & =& \frac{1}{2{\sigma }^2}(\sum _{i=1}^n{y}_i-\alpha \sum _{i=1}^{n}1-\beta \sum _{i=1}^n{x}_i)\\ & =& \frac{1}{2{\sigma }^2}(n\overline{y}-n\alpha -n\beta \overline{x})\cdots \overline{x}=\frac{1}{n}\sum _{i=1}^n{x}_i,\overline{y}=\frac{1}{n}\sum _{i=1}^n{y}_i\\ & =& \frac{n}{2{\sigma }^2}(\overline{y}-\alpha -\beta \overline{x})\end{array}$$ $$\begin{array}{rcl}\frac{\partial l}{\partial \beta }& =& \frac{\partial l}{\partial \beta }\{-\frac{1}{2{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\}\\ & =& -\frac{1}{2{\sigma }^2}\sum _{i=1}^n\frac{\partial l}{\partial \beta }{(y_i-\alpha -\beta x_i)}^2\\ & =& -\frac{1}{2{\sigma }^2}\sum _{i=1}^{n}2(y_i-\alpha -\beta x_i)(-x_i)\\ & =& -\frac{-2}{2{\sigma }^2}\sum _{i=1}^n(x_i{y}_i-x_i\alpha -\beta x_i^2)\\ & =& \frac{1}{{\sigma }^2}(\sum _{i=1}^n{x}_i{y}_i-\sum _{i=1}^n{x}_i\alpha -\sum _{i=1}^n\beta x_i^2)\\ & =& \frac{1}{{\sigma }^2}(\sum _{i=1}^n{x}_i{y}_i-n\overline{x}\alpha -\beta \sum _{i=1}^n{x}_i^2)\cdots \overline{x}=\frac{1}{n}\sum _{i=1}^n{x}_i\end{array}$$ $$\begin{array}{rcl}\frac{\partial l}{\partial {\sigma }^2}& =& \frac{\partial l}{\partial {\sigma }^2}\{-\frac{n}{2}\log \left({\sigma }^2\right)-\frac{1}{2{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\}\\ & =& -\frac{n}{2}\frac{\partial l}{\partial {\sigma }^2}\log \left({\sigma }^2\right)-\frac{1}{2}\left\{\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\right\}\frac{\partial l}{\partial {\sigma }^2}\frac{1}{{\sigma }^2}\\ & =& -\frac{n}{2}\frac{\partial l}{\partial {\sigma }^2}\log \left({\sigma }^2\right)-\frac{1}{2}\left\{\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\right\}\frac{\partial l}{\partial u}\frac{1}{u}\cdots u={\sigma }^2\\ & =& -\frac{n}{2}\frac{\partial l}{\partial {\sigma }^2}\log \left({\sigma }^2\right)-\frac{1}{2}\left\{\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\right\}\left(\frac{-1}{u^2}\right)\\ & =& -\frac{n}{2}\frac{1}{{\sigma }^2}-\frac{1}{2}\left\{\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\right\}\left(\frac{-1}{{\sigma }^4}\right)\cdots u={\sigma }^2\\ & =& -\frac{1}{2{\sigma }^2}\{n-\frac{1}{{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\}\end{array}$$

スコア凾数を連立させる

$$\begin{array}{r}\{\begin{array}{rcl}0& =& \frac{\partial l}{\partial \alpha }\\ 0& =& \frac{\partial l}{\partial \beta }\\ 0& =& \frac{\partial l}{\partial {\sigma }^2}\end{array}\end{array}$$ $$\begin{array}{r}\{\begin{array}{rcl}0& =& \frac{n}{2{\sigma }^2}(\overline{y}-\alpha -\beta \overline{x})\\ 0& =& \frac{1}{{\sigma }^2}(\sum _{i=1}^n{x}_i{y}_i-n\overline{x}\alpha -\beta \sum _{i=1}^n{x}_i^2)\\ 0& =& -\frac{1}{2{\sigma }^2}\{n-\frac{1}{{\sigma }^2}\sum _{i=1}^n{(y_i-\alpha -\beta x_i)}^2\}\end{array}\end{array}$$ $\alpha ,\beta ,{\sigma }^2$の推定量を$\hat{\alpha },\hat{\beta },{\hat{\sigma }}^2$とし，$\hat{\alpha },\hat{\beta },{\hat{\sigma }}^2$の最尤推定量(maximum likelihood estimator)を${\hat{\alpha }}_{ML},{\hat{\beta }}_{ML},{\hat{\sigma }}_{ML}^2$とする． $$\begin{array}{r}\{\begin{array}{rcl}0& =& \overline{y}-{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}\overline{x}\\ 0& =& \sum _{i=1}^n{x}_i{y}_i-n\overline{x}{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}\sum _{i=1}^n{x}_i^2\\ 0& =& n-\frac{1}{{\sigma }_{ML}^2}\sum _{i=1}^n{(y_i-{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}{x}_i)}^2\end{array}\end{array}$$

${\hat{\alpha }}_{ML}$を求める

$$\begin{array}{rcl}0& =& \overline{y}-{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}\overline{x}\\ {\hat{\alpha }}_{ML}& =& \overline{y}-{\hat{\beta }}_{ML}\overline{x}\end{array}$$

${\hat{\beta }}_{ML}$を求める

$$\begin{array}{rcl}0& =& \sum _{i=1}^n{x}_i{y}_i-n\overline{x}{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}\sum _{i=1}^n{x}_i^2\\ & =& \sum _{i=1}^n{x}_i{y}_i-n\overline{x}(\overline{y}-{\hat{\beta }}_{ML}\overline{x})-{\hat{\beta }}_{ML}\sum _{i=1}^n{x}_i^2\cdots {\hat{\alpha }}_{ML}=\overline{y}-{\hat{\beta }}_{ML}\overline{x}\\ & =& \sum _{i=1}^n{x}_i{y}_i-n\overline{x}\overline{y}+n{\hat{\beta }}_{ML}{\overline{x}}^2-{\hat{\beta }}_{ML}\sum _{i=1}^n{x}_i^2\\ & =& \sum _{i=1}^n{x}_i{y}_i-n\overline{x}\overline{y}-{\hat{\beta }}_{ML}\{\left(\sum _{i=1}^n{x}_i^2\right)-n{\overline{x}}^2\}\\ & =& \sum _{i=1}^n(x_i-\overline{x})(y_i-\overline{y})-{\hat{\beta }}_{ML}\sum _{i=1}^n{(x_i-\overline{x})}^2\\ & =& S_{xy}-{\hat{\beta }}_{ML}{S}_{xx}\cdots S_{xy}=\sum _{i=1}^n(x_i-\overline{x})(y_i-\overline{y}),S_{xx}=\sum _{i=1}^n(x_i-\overline{x}{)}^2\\ {\hat{\beta }}_{ML}& =& \frac{S_{xy}}{S_{xx}}\\ & =& \hat{\beta }\cdots \mathrm{正}\mathrm{規}\mathrm{方}\mathrm{程}\mathrm{式}\mathrm{の}\mathrm{解}\mathrm{と}\mathrm{同}\mathrm{じ}\\ {\hat{\alpha }}_{ML}& =& \overline{y}-{\hat{\beta }}_{ML}\overline{x}\\ & =& \overline{y}-\hat{\beta }\overline{x}\\ & =& \hat{\alpha }\cdots \mathrm{正}\mathrm{規}\mathrm{方}\mathrm{程}\mathrm{式}\mathrm{の}\mathrm{解}\mathrm{と}\mathrm{同}\mathrm{じ}\end{array}$$

${\hat{\sigma }}_{ML}^2$を求める

$$\begin{array}{rcl}0& =& n-\frac{1}{{\sigma }_{ML}^2}\sum _{i=1}^n{(y_i-{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}{x}_i)}^2\\ -n& =& -\frac{1}{{\sigma }_{ML}^2}\sum _{i=1}^n{(y_i-{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}{x}_i)}^2\\ -n{\sigma }_{ML}^2& =& -\sum _{i=1}^n{(y_i-{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}{x}_i)}^2\\ {\hat{\sigma }}_{ML}^2& =& \frac{1}{n}\sum _{i=1}^n{(y_i-{\hat{\alpha }}_{ML}-{\hat{\beta }}_{ML}{x}_i)}^2\\ & =& \frac{1}{n}\sum _{i=1}^n{(y_i-\hat{\alpha }-\hat{\beta }{x}_i)}^2\\ & =& \frac{1}{n}\sum _{i=1}^n{(y_i-{\hat{y}}_i)}^2\cdots {\hat{y}}_i=\hat{\alpha }+\hat{\beta }{x}_i\\ & =& \frac{1}{n}\sum _{i=1}^n{e}_i^2\\ & =& \frac{1}{n}(n-2)s^2\cdots \sum _{i=1}^n{e}_i^2=(n-2)s^2,s^2=\frac{1}{n-2}\sum _{i=1}^n{e}_i^2\\ & =& \frac{n-2}{n}{s}^2\end{array}$$

${\hat{\alpha }}_{ML},{\hat{\beta }}_{ML},{\hat{\sigma }}_{ML}^2$

以上より最尤推定量${\hat{\alpha }}_{ML},{\hat{\beta }}_{ML},{\hat{\sigma }}_{ML}^2$は以下のようになる． $$\begin{array}{rcl}{\hat{\beta }}_{ML}& =& \frac{S_{xy}}{S_{xx}}=\hat{\beta }\\ {\hat{\alpha }}_{ML}& =& \overline{y}-{\hat{\beta }}_{ML}\overline{x}=\overline{y}-\hat{\beta }\overline{x}=\hat{\alpha }\\ {\hat{\sigma }}_{ML}^2& =& \frac{n-2}{n}{s}^2\end{array}$$