SQL console consists of Schemas, Query, Action Output

SQL do not recognize upper or lower capital.

SQL Basic command

4 Language

데이터 정의어 Data Definition Language : make, modify, remove table

데이터 조작어 Data Manipulation Language : insert, index, modify, remove data

데이터 제어어 Data Control Language : access, remove data

트랜젝션 제어어 Transaction Control Language : execute, cancel DCL

DBA(DataBase Administrator) with DFL, DCL DA(Data Analyst) with DML, TCL

Data Definition Language

DataType

Each column should be defined as only one data type such as number, string, date, boolean.

number - BIT, INT, BIGINT, FLOAT, DOUBLE string - CHAR, NCHAR, VARCHAR, NVARCHAR date - DATETIME, DATE, TIME *DateBase sets bits to save data efficiently

restrict condition(제약 조건)

1. PK(PRIMARY KEY) : 중복되어 나타날 수 없는 단일 값 : NOT NULL

2. NOT NULL : NULL을 허용하지 않음

/********** DataBase **********/
/* Practice라는 이름으로 데이터베이스 생성 */
CREATE DATABASE Practice;

/* Practice 데이터베이스 사용 */
USE Practice;




/********** Table **********/
/* 회원테이블 생성 */
CREATE TABLE 회원테이블 (
회원번호 INT PRIMARY KEY,  /* Name DataType RestrictCondition */
이름 VARCHAR(20),         /* (바이트 수) */
가입일자 DATE NOT NULL,
수신동의 BIT
);

/* 테이블 명 변경 */
ALTER TABLE 회원테이블 RENAME 회원정보;

/* 회원테이블 조회 */
SELECT * FROM 회원테이블;

/* 테이블 삭제 */ 
DROP TABLE 회원정보;




 /********** Column **********/
 /* 성별 열 추가 */
 ALTER TABLE 회원테이블 ADD 성별 varchar(2);

 /* 성별 열의 타입 변경 */
 ALTER TABLE 회원테이블 MODIFY 성별 VARCHAR(20);

 /* 성별 -> 성 열의 이름 변경 */
 ALTER TABLE 회원테이블 CHANGE 성별 성 VARCHAR(2);

Data Manipulation Language

Insert, Find, rewrite, delete Data

USE Practice;

CREATE TABLE 회원테이블 (
회원번호 INT PRIMARY KEY,
이름 VARCHAR(20),
가입일자 DATE NOT NULL,
수신동의 BIT
);



/********** 데이터 삽입 **********/
INSERT INTO 회원테이블 VALUES (1001, '홍길동', '2020-01-02', 1);




/********** 제약조건 위반 **********/
/* PRIMARY KEY 위반 */
INSERT INTO 회원테이블 VALUES (1001, '짱구', '2020-01-06', 0);

/* NOT NULL 위반 */
INSERT INTO 회원테이블 VALUES (1002, '철수', NULL, 0);

/* 데이터타입 위반 */
INSERT INTO 회원테이블 VALUES (1003, '훈이', 1, 0);




/********** 데이터 조회 **********/
/* 모든 열 조회 */
SELECT * 
FROM 회원테이블;

/* 특정 열 조회 */
SELECT 회원번호, 이름
FROM 회원테이블;

/* 특정 열 이름을 변경하여 조회 */
SELECT 회원번호, 
이름 AS 성명
FROM 회원테이블;




/********** 데이터 수정 **********/
/* 모든 데이터 수정 */
UPDATE 회원테이블
SET 수신동의 = 0;

/* 특정 조건 데이터 수정 */
UPDATE 회원테이블
set 수신동의 = 1
WHERE 이름 = '홍길동';



/********** 데이터 삭제 **********/
/* 특정 데이터 삭제 */
DELETE
FROM 회원테이블
WHERE 이름 = '홍길동';

/* 모든 데이터 삭제 */
DELETE
FROM 회원테이블;

Data Control Language

DCL is used when DB admin give access right to usrs.

/********** 사용자 확인 **********/
/* MYSQL 데이터베이스 사용 */
USE MYSQL;



/********** 사용자 **********/
/* 사용자 아이디 및 비밀전호 생성 */
CREATE USER 'TEST'@LOCALHOST IDENTIFIED BY 'TEST';

/* 사용자 확인 */
SELECT *
FROM USER;

/* 사용자 비밀번호 변경 */
SET PASSWORD FOR 'TEST'@LOCALHOST = '1234';

/* 사용자 삭제 */
DROP USER 'TEST'@LOCALHOST;



/********** 권한 부여 및 제거 **********/
/* 권한 : CREATE, ALTER, DROP, INSERT, DELETE, UPDATE, SELECT 등 */

/* 특정 권한 부여 */
GRANT SELECT, DELETE ON Practice.회원테이블 TO 'TEST'@LOCALHOST;

/* 특정 권한 제거 */
REVOKE DELETE ON Practice.회원테이블 FROM 'TEST'@LOCALHOST;

/* 모든 권한 부여 */
GRANT ALL ON Practice.회원테이블 TO 'TEST'@LOCALHOST;

/* 모든 권한 제거 */
REVOKE ALL ON Practice.회원테이블 FROM 'TEST'@LOCALHOST;

Transaction Control Language

Transaction is undividable minimum unit. When error, roll back When no error, commit

USE Practice;


DROP TABLE 회원테이블;

CREATE TABLE 회원테이블 (
회원번호 INT PRIMARY KEY,
이름 VARCHAR(20),
가입일자 DATE NOT NULL,
수신동의 BIT
);

SELECT *
FROM 회원테이블;



/********** BEGIN + ROLLBACK **********/
/* 트랜젝션 시작 */
BEGIN;

/* 데이터 삽입 */
INSERT INTO 회원테이블 VALUES (1001, '홍길동', '2020-01-02', 1);

/* 취소 */
ROLLBACK;




/********** BEGIN + COMMIT **********/
/* 트랜젝션 시작 */
BEGIN;

/* 데이터 삽입 */
INSERT INTO 회원테이블 VALUES (1005, '장보고', '2020-01-02', 1);

/* 실행 */
COMMIT;




/********** 임시저장 SAVEPOINT **********/
/* 트랜젝션 시작 */
BEGIN;

/* 데이터 삽입 */
INSERT INTO 회원테이블 VALUES (1005, '임시저장', '2020-01-02', 1);

/* SAVEPOINT 지정 */
SAVEPOINT S1;

/* 1005 회원 이름 수정 */
UPDATE 회원테이블
SET 이름 = '임시저장 후';

/* SAVEPOINT 지정 */
SAVEPOINT S2;

/* 1005 회원 데이터 삭제 */
DELETE
FROM 회원테이블;

/* SAVEPOINT 지정 */
SAVEPOINT S3;

/* SAVEPOINT로 ROLLBACK */
ROLLBACK TO S2;

/* 실행 */
COMMIT;

!isHungry
isHungry ? "true" : "false"

Rest Operator ... converts the rest parameters to JavaScript array.

1. Without rest operator

   function printNums(num1, num2) {
   console.log(num1, num2);
   }
   

printNums(1, 2, 3, 4, 5);

1 2

<br/>

2. `arguments` is the object of all the parameters.

```javascript
function printNums(num1, num2) {
  console.log(arguments);
}

printNums(1, 2, 3, 4, 5);

[Arguments] { '0': 1, '1': 2, '2': 3, '3': 4, '4': 5 }

3. ... is combine the rest parameters as an object.

function printNums(num1, ...num2) {
  console.log(num1, num2);
}

printNums(1, 2, 3, 4, 5);

1 [ 2, 3, 4, 5 ]

• for ... in ... can access the key.

• for ... of ... can access the value.

for ... in ...

let arr = [10,20,30,40]

for (let val in arr) {
  console.log(val)
}

0
1
2
3

let obj = {
  a: 1,
  b: 2,
  c: 3,
};

for (let val in obj) {
  console.log(val);
}

a
b
c

With for ... in ..., you can get value using indexing.

let array = [10, 20, 30, 40];

for (let val in array) {
  console.log(array[val]);
}

10
20
30
40

for ... of ...

let array = [10, 20, 30, 40];

for (let val of array) {
  console.log(val);
}

10
20
30
40

let obj = {
  a: 1,
  b: 2,
  c: 3,
};

for (let val of obj) {
  console.log(val);
}

TypeError: obj is not iterable
    at Object.<anonymous> (/Users/basecamp/repo/ES6/es6.js:14:17)
    at Module._compile (node:internal/modules/cjs/loader:1103:14)
    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1157:10)
    at Module.load (node:internal/modules/cjs/loader:981:32)
    at Function.Module._load (node:internal/modules/cjs/loader:822:12)
    at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:77:12)
    at node:internal/main/run_main_module:17:47

String literal is a effective way to make string using defined variables.

const var1 = 'hello'
const var2 = 'world'

const StrLit = `${var1}, ${var2}!!`
console.log(StrLit)

hello world!!

JavaScript supports var, let, const to define variables.

However, you had better not to use var.

The reason is that

• var is restricted by the scope.
• var does not arise error even defined twice

So, you should use let instead of var for rewritable variables. And you should use const for constant variables.

var hello = 'hello'

if (true) {
  var hello = 'world'
  console.log(hello)
}

console.log(hello)

import numpy as np

a = np.array([1,2,3], dtype=int)
b = np.array([1.1,2.2,3.3], dtype=float)
c = np.array([1,1,0], dtype=bool)

print(a)
print(a.dtype)

print(b)
print(b.dtype)

print(c)
print(c.dtype)

[1 2 3]
int64
[1.1 2.2 3.3]
float64
[ True  True False]
bool

You can create an array using np.array({list}, dtype={int/float/bool}). And it can be returned by .dype property. If you omit dtype={} parameter, it automatically recognize the type of list.

Two dimensional array : Matrix

import numpy as np

a = np.array([[1,2,3],[4,5,6],[7,8,9]])

print(a)

[[1 2 3]
 [4 5 6]
 [7 8 9]]

The list for the multi dimensional array is composed of overlapped lists such as [[1,2,3],[4,5,6],[7,8,9]]. From that the

Multi dimensional array

import numpy as np

a = np.array([[[1,2],[3,4]],[[5,6],[7,8]],[[9,10],[11,12]]])

print(a)

[[[ 1  2]
  [ 3  4]]

 [[ 5  6]
  [ 7  8]]

 [[ 9 10]
  [11 12]]]

The list for the multi dimensional array is composed of multi overlapped lists such as [[[1,2],[3,4]],[[5,6],[7,8]],[[9,10],[11,12]]]. It is printed being seperated by Matrix .

Attribute of array

import numpy as np

a = np.array([[1,2,3],[4,5,6],[7,8,9]])
b = np.array([[[1,2],[3,4]],[[5,6],[7,8]],[[9,10],[11,12]]])

print(a.ndim)
print(a.shape)
print(a.dtype)
print()

print(b.ndim)
print(b.shape)
print(b.dtype)

2
(3, 3)
int64

3
(3, 2, 2)
int64

.ndim returns the number of dimension of array. .shape returns the shape of array. .dtype returns the data type of array.

Install the module

conda install numpy

# If success, there would be no response.

Import the module

import numpy as np

# If success, there would be no response.

Make array from list

import numpy as np

a_list = [1,2,3,4,5]
a_np = np.array(a_list)

print(a_list)
print(a_np)

[1, 2, 3, 4, 5]
[1 2 3 4 5]

There is a difference between list and array. The element of list is divided by comma. In contrast, the element of array is divided by space.

Copy array

import numpy as np

a_np = np.array([1,2,3,4,5])
b_np = a_np
c_np = a_np.copy()

b_np[0] = 99

print(a_np)
print(b_np)
print(c_np)

[99  2  3  4  5]
[99  2  3  4  5]
[1 2 3 4 5]

copy() Method duplicates the original array. This copy is not a reference but a seperate.

Indexing, Slicing array

import numpy as np

a_np = np.array([1,2,3,4,5])

print(a_np[0])
print(a_np[0:2])

1
[1 2]

Array can be indexed and sliced as similar to list.

Broadcast array

import numpy as np

a_np = np.array([1,2,3,4,5])
b_np = np.array([1,2,3,4,5])
c_np = np.array([1,2,3])

print(a_np + 2)
print(a_np - 2)
print(a_np * 2)
print(a_np / 2)

[3 4 5 6 7]
[-1  0  1  2  3]
[ 2  4  6  8 10]
[0.5 1.  1.5 2.  2.5]

By numpy. we can operate calculation between vector and scala. This is one of the differences from list.

Operate array

import numpy as np

a_np = np.array([1,2,3,4,5])
b_np = np.array([1,2,3,4,5])
c_np = np.array([1,2,3])

print(a_np + b_np)
print(a_np - b_np)
print(a_np * b_np)
print(a_np / b_np)

[ 2  4  6  8 10]
[0 0 0 0 0]
[ 1  4  9 16 25]
[1. 1. 1. 1. 1.]

By numpy, we can operate calculations between vectors. In that, the elements of operands should be in match of each.

Fisrt of all, Can you answer the below question?

What is pythonic?

Maybe, Pythonic means python-friendly.

And then, what about this question?

How can we code scripts pythonically?

In fact, there is a guideline which a lot of people agreed. It is call PEP8. But, it is not like we should follow it literally such as tax laws.

I think Python is one of the most intuitive language.

As known, Algorithm is a kind of procedure which contains special intensions. From this, we can realize that good algorithm comes from the systematic sequence of tasks.

Human recognize the procedure

Softmax is userd for multinominal classification.

The number of classifiers of the classification comes from the number of the neurons in the last layer.

And this is the number of the outputs as probability.

In Tensorflow, Softmax is a kind of Activation-Function.

Each output for the neuron of the last layer can be recognized as a logit.

$$ (\overrightarrow{p})^{T} :\ S_i((\overrightarrow{l})^{T}) = p_i = {{e^{l_i}} \over {\sum_{k=1}^{K} [e^{l_k}]}} :\ (\overrightarrow{l})^{T} $$

In softmax, the logit vector is converted to the probability vector. And the sum of the element of the probability vector equals 1.

For binary classification, there are two ways. One is to use Sigmoid with the last one neuron, another is to use Softmax with the last two neurons.

Actually, there is only Affine-Function and no Activation-Function at output layer. Instead of that, Softmax converts the vector z to the vector p

$$ (\overrightarrow{p})^{T} :\ (\overrightarrow{z})^{[O]} :\ (\overrightarrow{a})^{[O-1]} $$

In conclusion, Softmax is converter from logits to probabilities, Sigmoid is converter from a logit to aprobability.

Softmax Layer

$$ \hat{Y}^{T} :\ Softmax :\ L^{[O]} :\ L^{[I]} :\ X^T $$

In Deep-Learning process, learn depends on loss. loss is the scale of the difference between the dataset which is prepared and the output from the model.

sigmoid, softmax is mainly used for image classification.

There are reasons why starters should focus on the image classification. One is that it is easier than the other DL processes. Another is that there are lots of references because image classification is one of the first fields.

Odds

Odds are the ratio of the probability of one event to that of an alternative event. So, if odds are bigger than 1, the probability of one event is more than that of an alternative event. In contrast, if odds are smaller than 1, the probability of one event is less than that of an alternative event.

$$ o = {{p} \over {1-p}} $$

This is just another way to express the probability of an event happening or not. And, the closer the probability goes to 1, the odds diverge infinitely.

Logit

However, the graph of odds are not symmetric. So, it is required to put log on odds. This is called logit.

$$ l = log({{p} \over {1-p}}) \ \quad\quad\quad\quad = log(p) - log(1-p) $$

Logit is symmetric with respect to 0.5 at which odds are 1 and logit is 0. And logit diverges not only positively but also negatively.

Logit and Sigmoid

The inverse function of logit is the Sigmoid equation which returns probability from logit. $$ p = {{1} \over {1+{e^{-l}}}} \quad (0 \le p \le 1) $$ Sigmoid is symmetric with respect to 0 and there are horizontal asymptotes on 0 and on 1.

As logit diverges, the value of Affine-Transfromation also diverges. Because of this, We can deal with the value of Affine-Transfromation as a kind of logit. In other words, if the value of Affine-Transfromation passes through the sigmoid equation, it would be converted to the probability. Therefore, in this process, Deep Learning learns the probability.

Sigmoid is used for binary classification. Softmax is used for multinominal classification.

Logistic Regression is based on this.

$$ \overrightarrow{p} ;\ ;\ \overrightarrow{z}^{[1]} ;\ ;\ X^T $$

Affine-Function -> logit Activation-Function -> probability

Special Method, which is also called Magic Method, is a kind of method that can be set in user-defined class objects. It is usually surrounded by two Under-bars at head and tail of the keyword like __init__, __str__, etc.

Why do we need this?

With Spcial Method, your script must be more pythonic and more efficient.

You can

• Use the instance of the class as built-in types, such as data-types, functions, etc.

How to use Special Method

Take a look at the below code. At the bottom, I commanded to print() the parameter with the identifiter japan. At this, the japan is the instance of the class which I defined. So, the code prints the infomation of the type about the japan.

class country:
  def __init__(self, name, population):
    self.name = name
    self.population = population

japan = country('Japan', '125m')

print(japan)

# <__main__.country object at 0x7f0d90322290>

As Known, this is the expected output. However, there will be something different with Special Method.

class country:
  def __init__(self, name, population):
    self.name = name
    self.population = population

  # Add Specical Method inside the class
  def __str__(self):
    return f'name : {self.name}, population : {self.population}'

japan = country('Japan', '125m')

print(japan)

# name : Japan, population : 125m

At the bottom, I also commanded to print() the parameter with the identifiter japan. And then, the code prints the string from return of instance method '__str__()' inside the class japan.

Therefore, you can get string from class, just as parameter.

class country:
  def __init__(self, name, population):
    self.name = name
    self.population = population

japan = country('Japan', '125m')

japan()

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-3-88c499f4f664> in <module>()
      6 japan = country('Japan', '125m')
      7 
----> 8 japan()

TypeError: 'country' object is not callable

class country:
  def __init__(self, name, population):
    self.name = name
    self.population = population

  def __call__(self):
    print(f'name : {self.name}, population : {self.population}')

japan = country('Japan', '125m')

japan()

name : Japan, population : 125m

shape?

calculation?

output?

$l_{\overrightarrow{v}}$ : The length of v Vector

$n_{\overrightarrow{v}}$ : The number of v Vector

$i_{(l_{\overrightarrow{v}})}$ : The index of (the length of v Vector)

$m^{[from]}_{coordinate}$ : The element of M Marix

The length of

$$ {A}^{T} = (, \ddots {{a}{(i{(N_{\overrightarrow{x}})}, :i_{(l_{\overrightarrow{a}})})}^{[i_{L}]}}\ddots ,) \in : {\R}^{N_{\overrightarrow{x}} \times l_{\overrightarrow{\nu}^{[i_{(L)}]}}}

:\ :\

\uparrow\ Dense,Layer \ |\

:\ :\

{X}^{T} = (, \ddots {x}^{[i_{(N_{\overrightarrow{x}})}]}{i{(l_{\overrightarrow{x}})}}\ddots ,) \in : {\R}^{N_{\overrightarrow{x}} \times l_{\overrightarrow{x}}} $$

$$ {N_{\overrightarrow{x}} \times ( l_{\overrightarrow{x}}} \rightarrow l_{\overrightarrow{\nu}^{[i_{(L)}]}} ) $$

Passing through the layers, Dense Layer converts the columns of the Materix, from the number of x Vector to the number of $\nu$ Vector.

$$ z^{[1]}{i,: j} = (\overrightarrow{x}^{(i)})^T \cdot\overrightarrow{w}^{[1]}{j} + b^{[1]}_{j} $$

$$ a^{[1]}{i,: j} = g(z^{[1]}{i,: j}) $$

You have to observe the shape.

The Activation is one-to-one correspondence. The shape do not change.

So. in a column, as neuron-wise, the same neuron And, in a row, as Batch-wise, the same layer

Batch is the set of data, X Matrix

Cascaded Dense Layer

$$ X^T = \R^{n_{\overrightarrow{x}} \times l_{\overrightarrow{x}}} $$

$$ W = \R^{l_{\overrightarrow{x}} \times n_{\overrightarrow{\nu}}} :\ B = \R^{n_{\overrightarrow{x}} \times n_{\overrightarrow{\nu}}} $$

$$ Z^T = \R^{n_{\overrightarrow{x}} \times n_{\overrightarrow{\nu}}} $$

$$ A^T = \R^{n_{\overrightarrow{x}} \times n_{\overrightarrow{\nu}}} $$

I think that it is much more proper to count neurons using not length but number because neurons are indenpendent each other like the relations among the x vectors of the X matrix.

Cascaded structure.

Mxnet is the framework which controls multi GPUs at once. Because there are Include-Top classifier which is parameter in Mxnet, the drawings of the Dense Layers is better to be drawn from bottom to top.

The D

$$ (\overrightarrow{a}^{[l_L]})^{T} \in \R^{1 \times {l_{\nu^{[l_L]}}}}\ :\ \vdots\ :\ (\overrightarrow{a}^{[2]})^{T} \in \R^{1 \times {l_{\nu^{[2]}}}}\ :\ \uparrow \ L_2 = (\dots : \nu^{[2]}{i} : \dots) \ |\ : \ (\overrightarrow{a}^{[1]})^{T} \in \R^{1 \times {l{\nu^{[1]}}}}\ : \ \uparrow \ L_1 = (\dots : \nu^{[1]}_{i} : \dots) \ | \ : \ (\overrightarrow{x})^{T} \in \R^{1 \times {l_x}} $$

The number of parameter is

$$(l_x \times l_{\nu}) + (1 \times l_{\nu})$$

$$(l_x+1) \times l_{\nu}$$

In Dense Layer, as the network keep passing through the layers, the number of parameter considerably inceases.

Forward Propagation of The Second Dense Layer

The Second Dense Layer

Generalized Dense Layer

[ Generalized Dense Layer ]

$$ \vdots\ (\overrightarrow{a}^{[i]})^{T} \in \R^{1 \times {l_{\nu^{[i]}}}}\ :\ \uparrow \ L_i = (\dots : \nu^{[i]}{i} : \dots) \ |\ : \ (\overrightarrow{a}^{[i-1]})^{T} \in \R^{1 \times {l{\nu^{[i-1]}}}}\ \vdots\ $$

$$ (\overrightarrow{x})^T = (x_1 \dots x_i) $$

$$ \dots : {L}_i : \dots $$

$$ \dots : {\nu}_{i}^{:[i_L]} : \dots $$

$$ \dots : \overrightarrow{w}{i,: i{\nu}}^{[i_L]} : \dots \in \R^{l_w \times 1} $$ $$ : \dots : {b}{: : i{\nu}}^{[i_L]} : \dots \in \R^{1 \times 1} $$

There are two reasons why the weights is arrayed in column vector : One is that Algebra sets column vecters as the default, the other is that especially dense layer read the vector of weights in the column type.

Weighted Matrix and Bias Vector

combine the column vectors of the weights as matrix.

The shape of the weight matrix is that the length of Input times the length of Output.

$$ {W}^{[i_L]} \in \R^{l_w \times l_{\nu}} $$ $$ : \overrightarrow{b}^{[i_L]} \in \R^{1 \times l_{\nu}} $$

Forward Propagation of Dense Layer

$$ {a}i^{[i_L]} = \nu_i^{:[i_L]}((\overrightarrow{x})^T; : \overrightarrow{w}{i,: i_{\nu}}^{[i_L]}, : b_{: : i_{\nu}}^{[i_L]}) $$

$$ (\overrightarrow{a}^{[i_L]})^T = (\overrightarrow{x})^T \cdot {W}^{[i_L]} + \overrightarrow{b}^{[i_L]} $$

$$ (\overrightarrow{x})^T \in {\R}^{1 \times {l_x}} $$

$$ (\overrightarrow{a})^T \in {\R}^{1 \times {l_{\nu}}} $$

0
[1, 2, 3]
[4, 5, 6]
[7, 8, 9]

1
[10, 11, 12]
[13, 14, 15]
[16, 17, 18]

aaa = [[1,2,3],[4,5,6],[7,8,9]],[[10,11,12],[13,14,15],[16,17,18]]

for i, row in enumerate(aaa):
    print(i)
    print(*row, sep='\n')
    print()

0
[1, 2, 3]
[4, 5, 6]
[7, 8, 9]

1
[10, 11, 12]
[13, 14, 15]
[16, 17, 18]

Layer is the vector of Neurons, which are kinds of Parametric Function and generally have their own weight and their own bias for Featuring(Filtering).

Filterbank is a bundle of filters such as music equalizer. Because each filter of the bank is not affected by the input, the

Deep learning is based on the cascadee-structured filterbank

How a deep learning architecture works is making a filterbank with bundles of correlated neurons in the cascaded structure.

Dense Layer

The classification of the layer type depends on how the elements of the input are picked for passing through the layers.

In Dense layer, all the elements of the input is connected to the each neuron of the layers.

Network composes of Edge and Node.

Layer : $L^{[i]}$ ($L$ for layer) Neuron Vector : $\overrightarrow{\nu}^{[i]}$ # of Neurons : $l_i$ ($l$ for length)

$$ \overrightarrow{a}^{[i-1]} $$

$$ \overrightarrow{\nu}^{[i]}(\overrightarrow{a}^{[i-1]}) $$

$$ \overrightarrow{a}^{[i]} $$

'123'

# join() only can combine string.
aaa = ['1','2','3']
''.join(aaa)

1 + '1'

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/var/folders/y1/pfjkj5cd0wz3vhn094nv_yl80000gn/T/ipykernel_18846/2877782753.py in <module>
----> 1 1 + '1'

TypeError: unsupported operand type(s) for +: 'int' and 'str'

# In Activation layer, only activation

import tensorflow as tf

from tensorflow.math import exp, maximum
from tensorflow.keras.layers import Activation  # Activation is a kind of Layers

x = tf.random.normal(shape=(1, 5)) # input setting

# imp. activation function
sigmoid = Activation('sigmoid')
tanh = Activation('tanh')
relu = Activation('relu')

# forward propagation(TensorFlow)
y_sigmoid_tf = sigmoid(x)
y_tanh_tf = tanh(x)
y_relu_tf = relu(x)

# forward propagation(manual)
y_sigmoid_man = 1 / (1 + exp(-x))
y_tanh_man = (exp(x) - exp(-x)) / (exp(x) + exp(-x))
y_relu_man = maximum(x, 0)


print(f'x: {x.shape}\n{x.numpy()}')
print()
print(f'Sigmoid(TensorFlow): {y_sigmoid_tf.shape}\n{y_sigmoid_tf.numpy()}')
print(f'Sigmoid(TensorFlow): {y_sigmoid_man.shape}\n{y_sigmoid_man.numpy()}')
print()
print(f'Tanh(TensorFlow): {y_tanh_tf.shape}\n{y_tanh_tf.numpy()}')
print(f'Tanh(TensorFlow): {y_tanh_man.shape}\n{y_tanh_man.numpy()}')
print()
print(f'ReLU(TensorFlow): {y_relu_tf.shape}\n{y_relu_tf.numpy()}')
print(f'ReLU(TensorFlow): {y_relu_man.shape}\n{y_relu_man.numpy()}')

x: (1, 5)
[[ 0.4140593  -1.0886137  -1.9466102   1.297912   -0.11210307]]

Sigmoid(TensorFlow): (1, 5)
[[0.6020608  0.2518794  0.12492344 0.78548336 0.47200355]]
Sigmoid(TensorFlow): (1, 5)
[[0.60206085 0.25187942 0.12492345 0.78548336 0.47200358]]

Tanh(TensorFlow): (1, 5)
[[ 0.39191395 -0.79637164 -0.96005476  0.8611846  -0.1116358 ]]
Tanh(TensorFlow): (1, 5)
[[ 0.39191392 -0.7963717  -0.96005493  0.8611847  -0.1116358 ]]

ReLU(TensorFlow): (1, 5)
[[0.4140593 0.        0.        1.297912  0.       ]]
ReLU(TensorFlow): (1, 5)
[[0.4140593 0.        0.        1.297912  0.       ]]

Code.1-3-2: Activation in Dense Layer

# In Dense layer, affine + activation

import tensorflow as tf
from tensorflow.math import exp
from tensorflow.keras.layers import Dense   # Dense is a kind of Layers

x = tf.random.normal(shape=(1, 5)) # input setting

# imp. artificial neuron
# make a unit of dense layer in combination of the activation
dense_sigmoid = Dense(units=1, activation='sigmoid')
dense_tanh = Dense(units=1, activation='tanh')
dense_relu = Dense(units=1, activation='relu')

# forward propagation(tensorflow)
y_sigmoid = dense_sigmoid(x)
y_tanh = dense_tanh(x)
y_relu = dense_relu(x)

print(f'x: {x.shape}\n{x.numpy()}')
print()
print(f'AN with Sigmoid: {y_sigmoid.shape}\n{y_sigmoid.numpy()}')
print(f'AN with Tanh: {y_tanh.shape}\n{y_tanh.numpy()}')
print(f'AN with ReLU: {y_relu.shape}\n{y_relu.numpy()}')

print()
print('======')
print()

# forward progataion(manual)
W, B = dense_sigmoid.get_weights()
z = tf.linalg.matmul(x, W) + B
a = 1 / (1 + exp(-z))

print(f'Activation value(TensorFlow): {y_sigmoid.shape}\n{y_sigmoid.numpy()}')
print(f'Activation value(manual): {a.shape}\n{a.numpy()}')

x: (1, 5)
[[ 1.1621749 -0.7989249  2.0781152 -0.9444862  0.6221622]]

AN with Sigmoid: (1, 1)
[[0.7378292]]
AN with Tanh: (1, 1)
[[-0.7343226]]
AN with ReLU: (1, 1)
[[0.]]

======

Activation value(TensorFlow): (1, 1)
[[0.7378292]]
Activation value(manual): (1, 1)
[[0.73782927]]